Exemplo n.º 1
0
// GetDistro loads the task's distro and sends it to the requester.
func (as *APIServer) GetDistro(w http.ResponseWriter, r *http.Request) {
	task := MustHaveTask(r)

	// Get the distro for this task
	h, err := host.FindOne(host.ByRunningTaskId(task.Id))
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, err)
		return
	}

	// Fall back to checking host field on task doc
	if h == nil && len(task.HostId) > 0 {
		h, err = host.FindOne(host.ById(task.HostId))
		if err != nil {
			as.LoggedError(w, r, http.StatusInternalServerError, err)
			return
		}
		h.SetRunningTask(task.Id, h.AgentRevision, h.TaskDispatchTime)
	}

	if h == nil {
		message := fmt.Errorf("No host found running task %v", task.Id)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	// agent can't properly unmarshal provider settings map
	h.Distro.ProviderSettings = nil
	as.WriteJSON(w, http.StatusOK, h.Distro)
}
Exemplo n.º 2
0
func (as *APIServer) StartTask(w http.ResponseWriter, r *http.Request) {
	task := MustHaveTask(r)

	if !getGlobalLock(r.RemoteAddr, task.Id) {
		as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout)
		return
	}
	defer releaseGlobalLock(r.RemoteAddr, task.Id)

	evergreen.Logger.Logf(slogger.INFO, "Marking task started: %v", task.Id)

	taskStartInfo := &apimodels.TaskStartRequest{}
	if err := util.ReadJSONInto(r.Body, taskStartInfo); err != nil {
		http.Error(w, fmt.Sprintf("Error reading task start request for %v: %v", task.Id, err), http.StatusBadRequest)
		return
	}

	if err := task.MarkStart(); err != nil {
		message := fmt.Errorf("Error marking task '%v' started: %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	h, err := host.FindOne(host.ByRunningTaskId(task.Id))
	if err != nil {
		message := fmt.Errorf("Error finding host running task %v: %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	// Fall back to checking host field on task doc
	if h == nil && len(task.HostId) > 0 {
		evergreen.Logger.Logf(slogger.DEBUG, "Falling back to host field of task: %v", task.Id)
		h, err = host.FindOne(host.ById(task.HostId))
		if err != nil {
			as.LoggedError(w, r, http.StatusInternalServerError, err)
			return
		}
		h.SetRunningTask(task.Id, h.AgentRevision, h.TaskDispatchTime)
	}

	if h == nil {
		message := fmt.Errorf("No host found running task %v", task.Id)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	if err := h.SetTaskPid(taskStartInfo.Pid); err != nil {
		message := fmt.Errorf("Error calling set pid on task %v : %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}
	as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Task %v started on host %v", task.Id, h.Id))
}
Exemplo n.º 3
0
// GetDistro loads the task's distro and sends it to the requester.
func (as *APIServer) GetDistro(w http.ResponseWriter, r *http.Request) {
	task := MustHaveTask(r)

	// Get the distro for this task
	h, err := host.FindOne(host.ByRunningTaskId(task.Id))
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, err)
		return
	}

	// agent can't properly unmarshal provider settings map
	h.Distro.ProviderSettings = nil
	as.WriteJSON(w, http.StatusOK, h.Distro)
}
Exemplo n.º 4
0
func (as *APIServer) StartTask(w http.ResponseWriter, r *http.Request) {
	if !getGlobalLock(APIServerLockTitle) {
		as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout)
		return
	}
	defer releaseGlobalLock(APIServerLockTitle)

	task := MustHaveTask(r)

	evergreen.Logger.Logf(slogger.INFO, "Marking task started: %v", task.Id)

	taskStartInfo := &apimodels.TaskStartRequest{}
	if err := util.ReadJSONInto(r.Body, taskStartInfo); err != nil {
		http.Error(w, fmt.Sprintf("Error reading task start request for %v: %v", task.Id, err), http.StatusBadRequest)
		return
	}

	if err := task.MarkStart(); err != nil {
		message := fmt.Errorf("Error marking task '%v' started: %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	host, err := host.FindOne(host.ByRunningTaskId(task.Id))
	if err != nil {
		message := fmt.Errorf("Error finding host running task %v: %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	if host == nil {
		message := fmt.Errorf("No host found running task %v: %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	if err := host.SetTaskPid(taskStartInfo.Pid); err != nil {
		message := fmt.Errorf("Error calling set pid on task %v : %v", task.Id, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}
	as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Task %v started on host %v", task.Id, host.Id))
}
Exemplo n.º 5
0
// taskFinished constructs the appropriate response for each markEnd
// request the API server receives from an agent. The two possible responses are:
// 1. Inform the agent of another task to run
// 2. Inform the agent that it should terminate immediately
// The first case is the usual expected flow. The second case however, could
// occur for a number of reasons including:
// a. The version of the agent running on the remote machine is stale
// b. The host the agent is running on has been decommissioned
// c. There is no currently queued dispatchable and activated task
// In any of these aforementioned cases, the agent in question should terminate
// immediately and cease running any tasks on its host.
func (as *APIServer) taskFinished(w http.ResponseWriter, task *model.Task, finishTime time.Time) {
	taskEndResponse := &apimodels.TaskEndResponse{}

	// a. fetch the host this task just completed on to see if it's
	// now decommissioned
	host, err := host.FindOne(host.ByRunningTaskId(task.Id))
	if err != nil {
		message := fmt.Sprintf("Error locating host for task %v - set to %v: %v", task.Id,
			task.HostId, err)
		evergreen.Logger.Logf(slogger.ERROR, message)
		taskEndResponse.Message = message
		as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
		return
	}
	if host == nil {
		message := fmt.Sprintf("Error finding host running for task %v - set to %v", task.Id,
			task.HostId)
		evergreen.Logger.Logf(slogger.ERROR, message)
		taskEndResponse.Message = message
		as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
		return
	}
	if host.Status == evergreen.HostDecommissioned || host.Status == evergreen.HostQuarantined {
		markHostRunningTaskFinished(host, task, "")
		message := fmt.Sprintf("Host %v - running %v - is in state '%v'. Agent will terminate",
			task.HostId, task.Id, host.Status)
		evergreen.Logger.Logf(slogger.INFO, message)
		taskEndResponse.Message = message
		as.WriteJSON(w, http.StatusOK, taskEndResponse)
		return
	}

	// b. check if the agent needs to be rebuilt
	taskRunnerInstance := taskrunner.NewTaskRunner(&as.Settings)
	agentRevision, err := taskRunnerInstance.HostGateway.GetAgentRevision()
	if err != nil {
		markHostRunningTaskFinished(host, task, "")
		evergreen.Logger.Logf(slogger.ERROR, "failed to get agent revision: %v", err)
		taskEndResponse.Message = err.Error()
		as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
		return
	}
	if host.AgentRevision != agentRevision {
		markHostRunningTaskFinished(host, task, "")
		message := fmt.Sprintf("Remote agent needs to be rebuilt")
		evergreen.Logger.Logf(slogger.INFO, message)
		taskEndResponse.Message = message
		as.WriteJSON(w, http.StatusOK, taskEndResponse)
		return
	}

	// c. fetch the task's distro queue to dispatch the next pending task
	nextTask, err := getNextDistroTask(task, host)
	if err != nil {
		markHostRunningTaskFinished(host, task, "")
		evergreen.Logger.Logf(slogger.ERROR, err.Error())
		taskEndResponse.Message = err.Error()
		as.WriteJSON(w, http.StatusOK, taskEndResponse)
		return
	}
	if nextTask == nil {
		markHostRunningTaskFinished(host, task, "")
		taskEndResponse.Message = "No next task on queue"
	} else {
		taskEndResponse.Message = "Proceed with next task"
		taskEndResponse.RunNext = true
		taskEndResponse.TaskId = nextTask.Id
		taskEndResponse.TaskSecret = nextTask.Secret
		markHostRunningTaskFinished(host, task, nextTask.Id)
	}

	// give the agent the green light to keep churning
	as.WriteJSON(w, http.StatusOK, taskEndResponse)
}
Exemplo n.º 6
0
func TestBasicEndpoints(t *testing.T) {
	setupTlsConfigs(t)
	for tlsString, tlsConfig := range tlsConfigs {
		testTask, _, err := setupAPITestData(testConfig, "task", "linux-64", NoPatch, t)
		testutil.HandleTestingErr(err, t, "Couldn't make test data: %v", err)

		Convey("With a live api server, agent, and test task over "+tlsString, t, func() {
			testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
			testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
			testAgent, err := createAgent(testServer, testTask)
			testutil.HandleTestingErr(err, t, "failed to create agent: %v")

			Convey("sending logs should store the log messages properly", func() {
				msg1 := "task logger initialized!"
				msg2 := "system logger initialized!"
				msg3 := "exec logger initialized!"
				testAgent.logger.LogTask(slogger.INFO, msg1)
				testAgent.logger.LogSystem(slogger.INFO, msg2)
				testAgent.logger.LogExecution(slogger.INFO, msg3)
				time.Sleep(100 * time.Millisecond)
				testAgent.APILogger.FlushAndWait()

				// This returns logs in order of NEWEST first.
				logMessages, err := model.FindMostRecentLogMessages(testTask.Id, 0, 10, []string{}, []string{})
				testutil.HandleTestingErr(err, t, "failed to get log msgs")

				So(logMessages[2].Message, ShouldEndWith, msg1)
				So(logMessages[1].Message, ShouldEndWith, msg2)
				So(logMessages[0].Message, ShouldEndWith, msg3)
				Convey("Task endpoints should work between agent and server", func() {
					testAgent.StartBackgroundActions(&NoopSignalHandler{})
					Convey("calling GetTask should get retrieve same task back", func() {
						testTaskFromApi, err := testAgent.GetTask()
						So(err, ShouldBeNil)

						// ShouldResemble doesn't seem to work here, possibly because of
						// omitempty? anyways, just assert equality of the important fields
						So(testTaskFromApi.Id, ShouldEqual, testTask.Id)
						So(testTaskFromApi.Status, ShouldEqual, testTask.Status)
						So(testTaskFromApi.HostId, ShouldEqual, testTask.HostId)
					})

					Convey("calling start should flip the task's status to started", func() {
						err := testAgent.Start("1")
						testutil.HandleTestingErr(err, t, "Couldn't start task: %v", err)
						testTask, err := model.FindTask(testTask.Id)
						testutil.HandleTestingErr(err, t, "Couldn't refresh task from db: %v", err)
						So(testTask.Status, ShouldEqual, evergreen.TaskStarted)
						testHost, err := host.FindOne(host.ByRunningTaskId(testTask.Id))
						So(err, ShouldBeNil)
						So(testHost.Id, ShouldEqual, "testHost")
						So(testHost.RunningTask, ShouldEqual, testTask.Id)
					})

					Convey("calling end() should update task status properly", func() {
						commandType := model.SystemCommandType
						description := "random"
						details := &apimodels.TaskEndDetail{
							Description: description,
							Type:        commandType,
							TimedOut:    true,
							Status:      evergreen.TaskSucceeded,
						}
						testAgent.End(details)
						time.Sleep(100 * time.Millisecond)
						taskUpdate, err := model.FindTask(testTask.Id)
						So(err, ShouldBeNil)
						So(taskUpdate.Status, ShouldEqual, evergreen.TaskSucceeded)
						So(taskUpdate.Details.Description, ShouldEqual, description)
						So(taskUpdate.Details.Type, ShouldEqual, commandType)
						So(taskUpdate.Details.TimedOut, ShouldEqual, true)
					})

					Convey("no checkins should trigger timeout signal", func() {
						testAgent.idleTimeoutWatcher.SetDuration(2 * time.Second)
						testAgent.idleTimeoutWatcher.CheckIn()
						// sleep long enough for the timeout watcher to time out
						time.Sleep(3 * time.Second)
						timeoutSignal, ok := <-testAgent.signalHandler.idleTimeoutChan
						So(ok, ShouldBeTrue)
						So(timeoutSignal, ShouldEqual, IdleTimeout)
					})
				})
			})

		})
	}
}