// GetDistro loads the task's distro and sends it to the requester.
func (as *APIServer) GetDistro(w http.ResponseWriter, r *http.Request) {
task := MustHaveTask(r)
// Get the distro for this task
h, err := host.FindOne(host.ByRunningTaskId(task.Id))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
// Fall back to checking host field on task doc
if h == nil && len(task.HostId) > 0 {
h, err = host.FindOne(host.ById(task.HostId))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
h.SetRunningTask(task.Id, h.AgentRevision, h.TaskDispatchTime)
}
if h == nil {
message := fmt.Errorf("No host found running task %v", task.Id)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
// agent can't properly unmarshal provider settings map
h.Distro.ProviderSettings = nil
as.WriteJSON(w, http.StatusOK, h.Distro)
}
func (as *APIServer) StartTask(w http.ResponseWriter, r *http.Request) {
task := MustHaveTask(r)
if !getGlobalLock(r.RemoteAddr, task.Id) {
as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout)
return
}
defer releaseGlobalLock(r.RemoteAddr, task.Id)
evergreen.Logger.Logf(slogger.INFO, "Marking task started: %v", task.Id)
taskStartInfo := &apimodels.TaskStartRequest{}
if err := util.ReadJSONInto(r.Body, taskStartInfo); err != nil {
http.Error(w, fmt.Sprintf("Error reading task start request for %v: %v", task.Id, err), http.StatusBadRequest)
return
}
if err := task.MarkStart(); err != nil {
message := fmt.Errorf("Error marking task '%v' started: %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
h, err := host.FindOne(host.ByRunningTaskId(task.Id))
if err != nil {
message := fmt.Errorf("Error finding host running task %v: %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
// Fall back to checking host field on task doc
if h == nil && len(task.HostId) > 0 {
evergreen.Logger.Logf(slogger.DEBUG, "Falling back to host field of task: %v", task.Id)
h, err = host.FindOne(host.ById(task.HostId))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
h.SetRunningTask(task.Id, h.AgentRevision, h.TaskDispatchTime)
}
if h == nil {
message := fmt.Errorf("No host found running task %v", task.Id)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if err := h.SetTaskPid(taskStartInfo.Pid); err != nil {
message := fmt.Errorf("Error calling set pid on task %v : %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Task %v started on host %v", task.Id, h.Id))
}
// GetDistro loads the task's distro and sends it to the requester.
func (as *APIServer) GetDistro(w http.ResponseWriter, r *http.Request) {
task := MustHaveTask(r)
// Get the distro for this task
h, err := host.FindOne(host.ByRunningTaskId(task.Id))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
// agent can't properly unmarshal provider settings map
h.Distro.ProviderSettings = nil
as.WriteJSON(w, http.StatusOK, h.Distro)
}
func (as *APIServer) StartTask(w http.ResponseWriter, r *http.Request) {
if !getGlobalLock(APIServerLockTitle) {
as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout)
return
}
defer releaseGlobalLock(APIServerLockTitle)
task := MustHaveTask(r)
evergreen.Logger.Logf(slogger.INFO, "Marking task started: %v", task.Id)
taskStartInfo := &apimodels.TaskStartRequest{}
if err := util.ReadJSONInto(r.Body, taskStartInfo); err != nil {
http.Error(w, fmt.Sprintf("Error reading task start request for %v: %v", task.Id, err), http.StatusBadRequest)
return
}
if err := task.MarkStart(); err != nil {
message := fmt.Errorf("Error marking task '%v' started: %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
host, err := host.FindOne(host.ByRunningTaskId(task.Id))
if err != nil {
message := fmt.Errorf("Error finding host running task %v: %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if host == nil {
message := fmt.Errorf("No host found running task %v: %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if err := host.SetTaskPid(taskStartInfo.Pid); err != nil {
message := fmt.Errorf("Error calling set pid on task %v : %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Task %v started on host %v", task.Id, host.Id))
}
// taskFinished constructs the appropriate response for each markEnd
// request the API server receives from an agent. The two possible responses are:
// 1. Inform the agent of another task to run
// 2. Inform the agent that it should terminate immediately
// The first case is the usual expected flow. The second case however, could
// occur for a number of reasons including:
// a. The version of the agent running on the remote machine is stale
// b. The host the agent is running on has been decommissioned
// c. There is no currently queued dispatchable and activated task
// In any of these aforementioned cases, the agent in question should terminate
// immediately and cease running any tasks on its host.
func (as *APIServer) taskFinished(w http.ResponseWriter, task *model.Task, finishTime time.Time) {
taskEndResponse := &apimodels.TaskEndResponse{}
// a. fetch the host this task just completed on to see if it's
// now decommissioned
host, err := host.FindOne(host.ByRunningTaskId(task.Id))
if err != nil {
message := fmt.Sprintf("Error locating host for task %v - set to %v: %v", task.Id,
task.HostId, err)
evergreen.Logger.Logf(slogger.ERROR, message)
taskEndResponse.Message = message
as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
return
}
if host == nil {
message := fmt.Sprintf("Error finding host running for task %v - set to %v", task.Id,
task.HostId)
evergreen.Logger.Logf(slogger.ERROR, message)
taskEndResponse.Message = message
as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
return
}
if host.Status == evergreen.HostDecommissioned || host.Status == evergreen.HostQuarantined {
markHostRunningTaskFinished(host, task, "")
message := fmt.Sprintf("Host %v - running %v - is in state '%v'. Agent will terminate",
task.HostId, task.Id, host.Status)
evergreen.Logger.Logf(slogger.INFO, message)
taskEndResponse.Message = message
as.WriteJSON(w, http.StatusOK, taskEndResponse)
return
}
// b. check if the agent needs to be rebuilt
taskRunnerInstance := taskrunner.NewTaskRunner(&as.Settings)
agentRevision, err := taskRunnerInstance.HostGateway.GetAgentRevision()
if err != nil {
markHostRunningTaskFinished(host, task, "")
evergreen.Logger.Logf(slogger.ERROR, "failed to get agent revision: %v", err)
taskEndResponse.Message = err.Error()
as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
return
}
if host.AgentRevision != agentRevision {
markHostRunningTaskFinished(host, task, "")
message := fmt.Sprintf("Remote agent needs to be rebuilt")
evergreen.Logger.Logf(slogger.INFO, message)
taskEndResponse.Message = message
as.WriteJSON(w, http.StatusOK, taskEndResponse)
return
}
// c. fetch the task's distro queue to dispatch the next pending task
nextTask, err := getNextDistroTask(task, host)
if err != nil {
markHostRunningTaskFinished(host, task, "")
evergreen.Logger.Logf(slogger.ERROR, err.Error())
taskEndResponse.Message = err.Error()
as.WriteJSON(w, http.StatusOK, taskEndResponse)
return
}
if nextTask == nil {
markHostRunningTaskFinished(host, task, "")
taskEndResponse.Message = "No next task on queue"
} else {
taskEndResponse.Message = "Proceed with next task"
taskEndResponse.RunNext = true
taskEndResponse.TaskId = nextTask.Id
taskEndResponse.TaskSecret = nextTask.Secret
markHostRunningTaskFinished(host, task, nextTask.Id)
}
// give the agent the green light to keep churning
as.WriteJSON(w, http.StatusOK, taskEndResponse)
}
func TestBasicEndpoints(t *testing.T) {
setupTlsConfigs(t)
for tlsString, tlsConfig := range tlsConfigs {
testTask, _, err := setupAPITestData(testConfig, "task", "linux-64", NoPatch, t)
testutil.HandleTestingErr(err, t, "Couldn't make test data: %v", err)
Convey("With a live api server, agent, and test task over "+tlsString, t, func() {
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
Convey("sending logs should store the log messages properly", func() {
msg1 := "task logger initialized!"
msg2 := "system logger initialized!"
msg3 := "exec logger initialized!"
testAgent.logger.LogTask(slogger.INFO, msg1)
testAgent.logger.LogSystem(slogger.INFO, msg2)
testAgent.logger.LogExecution(slogger.INFO, msg3)
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
// This returns logs in order of NEWEST first.
logMessages, err := model.FindMostRecentLogMessages(testTask.Id, 0, 10, []string{}, []string{})
testutil.HandleTestingErr(err, t, "failed to get log msgs")
So(logMessages[2].Message, ShouldEndWith, msg1)
So(logMessages[1].Message, ShouldEndWith, msg2)
So(logMessages[0].Message, ShouldEndWith, msg3)
Convey("Task endpoints should work between agent and server", func() {
testAgent.StartBackgroundActions(&NoopSignalHandler{})
Convey("calling GetTask should get retrieve same task back", func() {
testTaskFromApi, err := testAgent.GetTask()
So(err, ShouldBeNil)
// ShouldResemble doesn't seem to work here, possibly because of
// omitempty? anyways, just assert equality of the important fields
So(testTaskFromApi.Id, ShouldEqual, testTask.Id)
So(testTaskFromApi.Status, ShouldEqual, testTask.Status)
So(testTaskFromApi.HostId, ShouldEqual, testTask.HostId)
})
Convey("calling start should flip the task's status to started", func() {
err := testAgent.Start("1")
testutil.HandleTestingErr(err, t, "Couldn't start task: %v", err)
testTask, err := model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Couldn't refresh task from db: %v", err)
So(testTask.Status, ShouldEqual, evergreen.TaskStarted)
testHost, err := host.FindOne(host.ByRunningTaskId(testTask.Id))
So(err, ShouldBeNil)
So(testHost.Id, ShouldEqual, "testHost")
So(testHost.RunningTask, ShouldEqual, testTask.Id)
})
Convey("calling end() should update task status properly", func() {
commandType := model.SystemCommandType
description := "random"
details := &apimodels.TaskEndDetail{
Description: description,
Type: commandType,
TimedOut: true,
Status: evergreen.TaskSucceeded,
}
testAgent.End(details)
time.Sleep(100 * time.Millisecond)
taskUpdate, err := model.FindTask(testTask.Id)
So(err, ShouldBeNil)
So(taskUpdate.Status, ShouldEqual, evergreen.TaskSucceeded)
So(taskUpdate.Details.Description, ShouldEqual, description)
So(taskUpdate.Details.Type, ShouldEqual, commandType)
So(taskUpdate.Details.TimedOut, ShouldEqual, true)
})
Convey("no checkins should trigger timeout signal", func() {
testAgent.idleTimeoutWatcher.SetDuration(2 * time.Second)
testAgent.idleTimeoutWatcher.CheckIn()
// sleep long enough for the timeout watcher to time out
time.Sleep(3 * time.Second)
timeoutSignal, ok := <-testAgent.signalHandler.idleTimeoutChan
So(ok, ShouldBeTrue)
So(timeoutSignal, ShouldEqual, IdleTimeout)
})
})
})
})
}
}