Example #1
0
// terminate the passed-in slice of hosts. returns any errors that occur
// terminating the hosts
func terminateHosts(hosts []host.Host, settings *evergreen.Settings, reason string) []error {
	errChan := make(chan error)
	for _, h := range hosts {
		evergreen.Logger.Logf(slogger.INFO, "Terminating host %v", h.Id)
		// terminate the host in a goroutine, passing the host in as a parameter
		// so that the variable isn't reused for subsequent iterations
		go func(hostToTerminate host.Host) {
			errChan <- func() error {
				event.LogMonitorOperation(hostToTerminate.Id, reason)
				err := util.RunFunctionWithTimeout(func() error {
					return terminateHost(&hostToTerminate, settings)
				}, 12*time.Minute)
				if err != nil {
					if err == util.ErrTimedOut {
						return fmt.Errorf("timeout terminating host %v", hostToTerminate.Id)
					}
					return fmt.Errorf("error terminating host %v: %v", hostToTerminate.Id, err)
				}
				evergreen.Logger.Logf(slogger.INFO, "Successfully terminated host %v", hostToTerminate.Id)
				return nil
			}()
		}(h)
	}
	var errors []error
	for range hosts {
		if err := <-errChan; err != nil {
			errors = append(errors, err)
		}
	}
	return errors
}
Example #2
0
// copyScript writes a given script as file "name" to the target host. This works
// by creating a local copy of the script on the runner's machine, scping it over
// then removing the local copy.
func (init *HostInit) copyScript(target *host.Host, name, script string) error {
	// parse the hostname into the user, host and port
	hostInfo, err := util.ParseSSHInfo(target.Host)
	if err != nil {
		return err
	}
	user := target.Distro.User
	if hostInfo.User != "" {
		user = hostInfo.User
	}

	// create a temp file for the script
	file, err := ioutil.TempFile("", name)
	if err != nil {
		return fmt.Errorf("error creating temporary script file: %v", err)
	}
	defer func() {
		file.Close()
		os.Remove(file.Name())
	}()

	expanded, err := init.expandScript(script)
	if err != nil {
		return fmt.Errorf("error expanding script for host %v: %v", target.Id, err)
	}
	if _, err := io.WriteString(file, expanded); err != nil {
		return fmt.Errorf("error writing local script: %v", err)
	}

	cloudHost, err := providers.GetCloudHost(target, init.Settings)
	if err != nil {
		return fmt.Errorf("failed to get cloud host for %v: %v", target.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return fmt.Errorf("error getting ssh options for host %v: %v", target.Id, err)
	}

	var scpCmdStderr bytes.Buffer
	scpCmd := &command.ScpCommand{
		Source:         file.Name(),
		Dest:           name,
		Stdout:         &scpCmdStderr,
		Stderr:         &scpCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}
	err = util.RunFunctionWithTimeout(scpCmd.Run, SCPTimeout)
	if err != nil {
		if err == util.ErrTimedOut {
			scpCmd.Stop()
			return fmt.Errorf("scp-ing script timed out")
		}
		return fmt.Errorf("error (%v) copying script to remote machine: %v",
			err, scpCmdStderr.String())
	}
	return nil
}
Example #3
0
// terminate the passed-in slice of hosts. returns any errors that occur
// terminating the hosts
func terminateHosts(hosts []host.Host, settings *evergreen.Settings) []error {

	// used to store any errors that occur
	var errors []error

	// for terminating the different hosts in parallel
	waitGroup := &sync.WaitGroup{}

	// to ensure thread-safe appending to the errors
	errsLock := &sync.Mutex{}

	for _, h := range hosts {

		evergreen.Logger.Logf(slogger.INFO, "Terminating host %v...", h.Id)

		waitGroup.Add(1)

		// terminate the host in a goroutine. pass the host in as a parameter
		// so that the variable isn't reused for subsequent iterations
		go func(hostToTerminate host.Host) {

			defer waitGroup.Done()

			// wrapper function to terminate the host
			terminateFunc := func() error {
				return terminateHost(&hostToTerminate, settings)
			}

			// run the function with a timeout
			err := util.RunFunctionWithTimeout(terminateFunc, 10*time.Second)

			if err == util.ErrTimedOut {
				errsLock.Lock()
				errors = append(errors, fmt.Errorf("timeout terminating"+
					" host %v", hostToTerminate.Id))
				errsLock.Unlock()
			} else if err != nil {
				errsLock.Lock()
				errors = append(errors, fmt.Errorf("error terminating host:"+
					" %v", err))
				errsLock.Unlock()
			} else {
				evergreen.Logger.Logf(slogger.INFO, "Successfully terminated host"+
					" %v", hostToTerminate.Id)
			}

		}(h)

	}

	// make sure all terminations finish
	waitGroup.Wait()

	return errors
}
Example #4
0
// Start the agent process on the specified remote host, and have it run
// the specified task.
// Returns an error if starting the agent remotely fails.
func (self *AgentBasedHostGateway) startAgentOnRemote(
	settings *evergreen.Settings, task *model.Task,
	hostObj *host.Host, sshOptions []string) error {

	// the path to the agent binary on the remote machine
	pathToExecutable := filepath.Join(hostObj.Distro.WorkDir, "main")

	// build the command to run on the remote machine
	remoteCmd := fmt.Sprintf(
		`%v -api_server "%v" -task_id "%v" -task_secret "%v" -log_prefix "%v" -https_cert "%v"`,
		pathToExecutable, settings.ApiUrl, task.Id, task.Secret, filepath.Join(hostObj.Distro.WorkDir,
			agentFile), settings.Expansions["api_httpscert_path"],
	)
	evergreen.Logger.Logf(slogger.INFO, "%v", remoteCmd)

	// compute any info necessary to ssh into the host
	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
	}

	// run the command to kick off the agent remotely
	var startAgentLog bytes.Buffer
	startAgentCmd := &command.RemoteCommand{
		CmdString:      remoteCmd,
		Stdout:         &startAgentLog,
		Stderr:         &startAgentLog,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     true,
	}

	// run the command to start the agent with a timeout
	err = util.RunFunctionWithTimeout(
		startAgentCmd.Run,
		StartAgentTimeout,
	)
	if err != nil {
		if err == util.ErrTimedOut {
			startAgentCmd.Stop()
			return fmt.Errorf("starting agent timed out")
		}
		return fmt.Errorf("error starting agent on host %v (%v): %v", hostObj.Id, err, startAgentLog)
	}

	return nil
}
Example #5
0
// RunRemoteScript executes a shell script that already exists on the remote host,
// returning logs and any errors that occur. Logs may still be returned for some errors.
func RunRemoteScript(h *host.Host, script string, sshOptions []string) (string, error) {
	// parse the hostname into the user, host and port
	hostInfo, err := util.ParseSSHInfo(h.Host)
	if err != nil {
		return "", err
	}
	user := h.Distro.User
	if hostInfo.User != "" {
		user = hostInfo.User
	}

	// run the remote script as sudo, if appropriate
	sudoStr := ""
	if h.Distro.SetupAsSudo {
		sudoStr = "sudo "
	}
	// run command to ssh into remote machine and execute script
	sshCmdStd := &util.CappedWriter{
		Buffer:   &bytes.Buffer{},
		MaxBytes: 1024 * 1024, // 1MB
	}
	cmd := &command.RemoteCommand{
		CmdString:      sudoStr + "sh " + script,
		Stdout:         sshCmdStd,
		Stderr:         sshCmdStd,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        []string{"-p", hostInfo.Port},
		Background:     false,
	}
	// force creation of a tty if sudo
	if h.Distro.SetupAsSudo {
		cmd.Options = []string{"-t", "-t", "-p", hostInfo.Port}
	}
	cmd.Options = append(cmd.Options, sshOptions...)

	// run the ssh command with given timeout
	err = util.RunFunctionWithTimeout(
		cmd.Run,
		SSHTimeout,
	)
	return sshCmdStd.String(), err
}
Example #6
0
func (init *HostInit) fetchRemoteTaskData(taskId, cliPath, confPath string, target *host.Host) error {
	hostSSHInfo, err := util.ParseSSHInfo(target.Host)
	if err != nil {
		return fmt.Errorf("error parsing ssh info %v: %v", target.Host, err)
	}

	cloudHost, err := providers.GetCloudHost(target, init.Settings)
	if err != nil {
		return fmt.Errorf("Failed to get cloud host for %v: %v", target.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return fmt.Errorf("Error getting ssh options for host %v: %v", target.Id, err)
	}
	sshOptions = append(sshOptions, "-o", "UserKnownHostsFile=/dev/null")

	/* TESTING ONLY
	setupDebugSSHTunnel(path_to_ssh_keys, target.User, hostSSHInfo.Hostname)
	*/

	// When testing, use this writer to force a copy of the output to be written to standard out so
	// that remote command failures also show up in server log output.
	//cmdOutput := io.MultiWriter(&util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}, os.Stdout)

	cmdOutput := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
	makeShellCmd := &command.RemoteCommand{
		CmdString:      fmt.Sprintf("%s -c '%s' fetch -t %s --source --artifacts --dir='%s'", cliPath, confPath, taskId, target.Distro.WorkDir),
		Stdout:         cmdOutput,
		Stderr:         cmdOutput,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-p", hostSSHInfo.Port}, sshOptions...),
	}

	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(makeShellCmd.Run, 15*time.Minute)
	return err
}
Example #7
0
// setupHost runs the specified setup script for an individual host. Returns
// the output from running the script remotely, as well as any error that
// occurs. If the script exits with a non-zero exit code, the error will be non-nil.
func (init *HostInit) setupHost(targetHost *host.Host) ([]byte, error) {

	// fetch the appropriate cloud provider for the host
	cloudMgr, err := providers.GetCloudManager(targetHost.Provider, init.Settings)
	if err != nil {
		return nil,
			fmt.Errorf("failed to get cloud manager for host %v with provider %v: %v",
				targetHost.Id, targetHost.Provider, err)
	}

	// mark the host as initializing
	if err := targetHost.SetInitializing(); err != nil {
		if err == mgo.ErrNotFound {
			return nil, ErrHostAlreadyInitializing
		} else {
			return nil, fmt.Errorf("database error: %v", err)
		}
	}

	// run the function scheduled for when the host is up
	err = cloudMgr.OnUp(targetHost)
	if err != nil {
		// if this fails it is probably due to an API hiccup, so we keep going.
		evergreen.Logger.Logf(slogger.WARN, "OnUp callback failed for host '%v': '%v'", targetHost.Id, err)
	}

	// run the remote setup script as sudo, if appropriate
	sudoStr := ""
	if targetHost.Distro.SetupAsSudo {
		sudoStr = "sudo "
	}

	// parse the hostname into the user, host and port
	hostInfo, err := util.ParseSSHInfo(targetHost.Host)
	if err != nil {
		return nil, err
	}
	user := targetHost.Distro.User
	if hostInfo.User != "" {
		user = hostInfo.User
	}

	// create a temp file for the setup script
	fileName := "setup.sh"
	file, err := ioutil.TempFile("", fileName)
	if err != nil {
		return nil, fmt.Errorf("error creating setup script: %v", err)
	}
	defer func() {
		file.Close()
		os.Remove(file.Name())
	}()

	// build the setup script
	setup, err := init.buildSetupScript(targetHost)
	if err != nil {
		return nil, fmt.Errorf("error building setup script for host %v: %v", targetHost.Id, err)
	}

	// write the setup script to the file
	if _, err := file.Write([]byte(setup)); err != nil {
		return nil, fmt.Errorf("error writing remote setup script: %v", err)
	}

	cloudHost, err := providers.GetCloudHost(targetHost, init.Settings)
	if err != nil {
		return nil, fmt.Errorf("Failed to get cloud host for %v: %v", targetHost.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return nil, fmt.Errorf("Error getting ssh options for host %v: %v", targetHost.Id, err)
	}

	// copy setup script over to the remote machine
	var scpSetupCmdStderr bytes.Buffer
	scpSetupCmd := &command.ScpCommand{
		Source:         file.Name(),
		Dest:           fileName,
		Stdout:         &scpSetupCmdStderr,
		Stderr:         &scpSetupCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}

	// run the command to scp the setup script with a timeout
	err = util.RunFunctionWithTimeout(
		scpSetupCmd.Run,
		SCPTimeout,
	)
	if err != nil {
		if err == util.ErrTimedOut {
			scpSetupCmd.Stop()
			return nil, fmt.Errorf("scp-ing setup script timed out")
		}
		return nil, fmt.Errorf("error (%v) copying setup script to remote "+
			"machine: %v", err, scpSetupCmdStderr.String())
	}

	// run command to ssh into remote machine and execute setup script
	var sshSetupCmdStderr bytes.Buffer
	runSetupCmd := &command.RemoteCommand{
		CmdString:      sudoStr + "sh " + fileName,
		Stdout:         &sshSetupCmdStderr,
		Stderr:         &sshSetupCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        []string{"-p", hostInfo.Port},
		Background:     false,
	}

	// only force creation of a tty if sudo
	if targetHost.Distro.SetupAsSudo {
		runSetupCmd.Options = []string{"-t", "-t", "-p", hostInfo.Port}
	}
	runSetupCmd.Options = append(runSetupCmd.Options, sshOptions...)

	// run the ssh command with given timeout
	err = util.RunFunctionWithTimeout(
		runSetupCmd.Run,
		time.Duration(SSHTimeoutSeconds)*time.Second,
	)

	return sshSetupCmdStderr.Bytes(), err
}
Example #8
0
// LoadClient places the evergreen command line client on the host, places a copy of the user's
// settings onto the host, and makes the binary appear in the $PATH when the user logs in.
// If successful, returns an instance of LoadClientResult which contains the paths where the
// binary and config file were written to.
func (init *HostInit) LoadClient(target *host.Host) (*LoadClientResult, error) {
	// Make sure we have the binary we want to upload - if it hasn't been built for the given
	// architecture, fail early
	cliBinaryPath, err := LocateCLIBinary(init.Settings, target.Distro.Arch)
	if err != nil {
		return nil, fmt.Errorf("couldn't locate CLI binary for upload: %v", err)
	}
	if target.ProvisionOptions == nil {
		return nil, fmt.Errorf("ProvisionOptions is nil")
	}
	if target.ProvisionOptions.OwnerId == "" {
		return nil, fmt.Errorf("OwnerId not set")
	}

	// get the information about the owner of the host
	owner, err := user.FindOne(user.ById(target.ProvisionOptions.OwnerId))
	if err != nil {
		return nil, fmt.Errorf("couldn't fetch owner %v for host: %v", target.ProvisionOptions.OwnerId, err)
	}

	// 1. mkdir the destination directory on the host,
	//    and modify ~/.profile so the target binary will be on the $PATH
	targetDir := "cli_bin"
	hostSSHInfo, err := util.ParseSSHInfo(target.Host)
	if err != nil {
		return nil, fmt.Errorf("error parsing ssh info %v: %v", target.Host, err)
	}

	cloudHost, err := providers.GetCloudHost(target, init.Settings)
	if err != nil {
		return nil, fmt.Errorf("Failed to get cloud host for %v: %v", target.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return nil, fmt.Errorf("Error getting ssh options for host %v: %v", target.Id, err)
	}
	sshOptions = append(sshOptions, "-o", "UserKnownHostsFile=/dev/null")

	mkdirOutput := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}

	// Create the directory for the binary to be uploaded into.
	// Also, make a best effort to add the binary's location to $PATH upon login. If we can't do
	// this successfully, the command will still succeed, it just means that the user will have to
	// use an absolute path (or manually set $PATH in their shell) to execute it.
	makeShellCmd := &command.RemoteCommand{
		CmdString:      fmt.Sprintf("mkdir -m 777 -p ~/%s && (echo 'PATH=$PATH:~/%s' >> ~/.profile || true; echo 'PATH=$PATH:~/%s' >> ~/.bash_profile || true)", targetDir, targetDir, targetDir),
		Stdout:         mkdirOutput,
		Stderr:         mkdirOutput,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-p", hostSSHInfo.Port}, sshOptions...),
	}

	scpOut := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(makeShellCmd.Run, 30*time.Second)
	if err != nil {
		return nil, fmt.Errorf("error running setup command for cli, %v: '%v'", mkdirOutput.Buffer.String(), err)
	}
	// place the binary into the directory
	scpSetupCmd := &command.ScpCommand{
		Source:         cliBinaryPath,
		Dest:           fmt.Sprintf("~/%s/evergreen", targetDir),
		Stdout:         scpOut,
		Stderr:         scpOut,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-P", hostSSHInfo.Port}, sshOptions...),
	}

	// run the command to scp the setup script with a timeout
	err = util.RunFunctionWithTimeout(scpSetupCmd.Run, 3*time.Minute)
	if err != nil {
		return nil, fmt.Errorf("error running SCP command for cli, %v: '%v'", scpOut.Buffer.String(), err)
	}

	// 4. Write a settings file for the user that owns the host, and scp it to the directory
	outputStruct := model.CLISettings{
		User:          owner.Id,
		APIKey:        owner.APIKey,
		APIServerHost: init.Settings.ApiUrl + "/api",
		UIServerHost:  init.Settings.Ui.Url,
	}
	outputJSON, err := json.Marshal(outputStruct)
	if err != nil {
		return nil, err
	}

	tempFileName, err := util.WriteTempFile("", outputJSON)
	if err != nil {
		return nil, err
	}
	defer os.Remove(tempFileName)

	scpYmlCommand := &command.ScpCommand{
		Source:         tempFileName,
		Dest:           fmt.Sprintf("~/%s/.evergreen.yml", targetDir),
		Stdout:         scpOut,
		Stderr:         scpOut,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-P", hostSSHInfo.Port}, sshOptions...),
	}
	err = util.RunFunctionWithTimeout(scpYmlCommand.Run, 30*time.Second)
	if err != nil {
		return nil, fmt.Errorf("error running SCP command for evergreen.yml, %v: '%v'", scpOut.Buffer.String(), err)
	}

	return &LoadClientResult{
		BinaryPath: fmt.Sprintf("~/%s/evergreen", targetDir),
		ConfigPath: fmt.Sprintf("~/%s/.evergreen.yml", targetDir),
	}, nil
}
Example #9
0
// Prepare the remote machine to run a task.
func (self *AgentBasedHostGateway) prepRemoteHost(settings *evergreen.Settings,
	hostObj host.Host, sshOptions []string, mciHome string) (string, error) {

	// compute any info necessary to ssh into the host
	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return "", fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
	}

	// first, create the necessary sandbox of directories on the remote machine
	makeShellCmd := &command.RemoteCommand{
		CmdString:      fmt.Sprintf("mkdir -m 777 -p %v", hostObj.Distro.WorkDir),
		Stdout:         ioutil.Discard, // TODO(EVG-233) change to real logging
		Stderr:         ioutil.Discard,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     false,
	}

	evergreen.Logger.Logf(slogger.INFO, "Directories command: '%#v'", makeShellCmd)

	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(
		makeShellCmd.Run,
		MakeShellTimeout,
	)
	if err != nil {
		// if it timed out, kill the command
		if err == util.ErrTimedOut {
			makeShellCmd.Stop()
			return "", fmt.Errorf("creating remote directories timed out")
		}
		return "", fmt.Errorf("error creating directories on remote machine: %v", err)
	}

	scpConfigsCmd := &command.ScpCommand{
		Source:         filepath.Join(mciHome, settings.ConfigDir),
		Dest:           hostObj.Distro.WorkDir,
		Stdout:         ioutil.Discard, // TODO(EVG-233) change to real logging
		Stderr:         ioutil.Discard,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options: append([]string{"-P", hostInfo.Port, "-r"},
			sshOptions...),
	}

	// run the command to scp the configs with a timeout
	err = util.RunFunctionWithTimeout(
		scpConfigsCmd.Run,
		SCPTimeout,
	)
	if err != nil {
		// if it timed out, kill the scp command
		if err == util.ErrTimedOut {
			scpConfigsCmd.Stop()
			return "", fmt.Errorf("scp-ing config directory timed out")
		}
		return "", fmt.Errorf("error copying config directory to remote: "+
			"machine %v", err)
	}

	// third, copy over the correct agent binary to the remote machine
	var scpAgentCmdStderr bytes.Buffer
	execSubPath, err := executableSubPath(hostObj.Distro.Id)
	if err != nil {
		return "", fmt.Errorf("error computing subpath to executable: %v", err)
	}
	scpAgentCmd := &command.ScpCommand{
		Source:         filepath.Join(self.ExecutablesDir, execSubPath),
		Dest:           hostObj.Distro.WorkDir,
		Stdout:         ioutil.Discard, // TODO(EVG-233) change to real logging
		Stderr:         &scpAgentCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}

	// get the agent's revision before scp'ing over the executable
	preSCPAgentRevision, err := self.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting pre scp agent "+
			"revision: %v", err)
	}

	// run the command to scp the agent with a timeout
	err = util.RunFunctionWithTimeout(
		scpAgentCmd.Run,
		SCPTimeout,
	)
	if err != nil {
		if err == util.ErrTimedOut {
			scpAgentCmd.Stop()
			return "", fmt.Errorf("scp-ing agent binary timed out")
		}
		return "", fmt.Errorf("error (%v) copying agent binary to remote "+
			"machine: %v", err, scpAgentCmdStderr.String())
	}

	// get the agent's revision after scp'ing over the executable
	postSCPAgentRevision, err := self.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting post scp agent "+
			"revision: %v", err)
	}

	if preSCPAgentRevision != postSCPAgentRevision {
		evergreen.Logger.Logf(slogger.WARN, "Agent revision was %v before scp "+
			"but is now %v. Using previous revision %v for host %v",
			preSCPAgentRevision, postSCPAgentRevision, preSCPAgentRevision,
			hostObj.Id)
	}

	return preSCPAgentRevision, nil
}
Example #10
0
// Prepare the remote machine to run a task.
// This consists of:
// 1. Creating the directories on the remote host where, according to the distro's settings,
//    the agent should be placed.
// 2. Copying the agent into that directory.
func (agbh *AgentHostGateway) prepRemoteHost(hostObj host.Host, sshOptions []string) (string, error) {
	// compute any info necessary to ssh into the host
	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return "", fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
	}

	// first, create the necessary sandbox of directories on the remote machine
	mkdirOutput := newCappedOutputLog()
	makeShellCmd := &command.RemoteCommand{
		Id:             fmt.Sprintf("agent_mkdir-%v", rand.Int()),
		CmdString:      fmt.Sprintf("mkdir -m 777 -p %v", hostObj.Distro.WorkDir),
		Stdout:         mkdirOutput,
		Stderr:         mkdirOutput,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     false,
	}

	evergreen.Logger.Logf(slogger.INFO, "Directories command: '%#v'", makeShellCmd)

	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(makeShellCmd.Run, MakeShellTimeout)
	if err != nil {
		// if it timed out, kill the command
		if err == util.ErrTimedOut {
			makeShellCmd.Stop()
			return "", fmt.Errorf("creating remote directories timed out: %v", mkdirOutput.String())
		}
		return "", fmt.Errorf(
			"error creating directories on remote machine (%v): %v", err, mkdirOutput.String())
	}

	// third, copy over the correct agent binary to the remote machine
	execSubPath, err := executableSubPath(hostObj.Distro.Id)
	if err != nil {
		return "", fmt.Errorf("error computing subpath to executable: %v", err)
	}

	scpAgentOutput := newCappedOutputLog()
	scpAgentCmd := &command.ScpCommand{
		Id:             fmt.Sprintf("scp%v", rand.Int()),
		Source:         filepath.Join(agbh.ExecutablesDir, execSubPath),
		Dest:           hostObj.Distro.WorkDir,
		Stdout:         scpAgentOutput,
		Stderr:         scpAgentOutput,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}

	// get the agent's revision before scp'ing over the executable
	preSCPAgentRevision, err := agbh.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting pre scp agent "+
			"revision: %v", err)
	}

	// run the command to scp the agent with a timeout
	err = util.RunFunctionWithTimeout(scpAgentCmd.Run, SCPTimeout)
	if err != nil {
		if err == util.ErrTimedOut {
			scpAgentCmd.Stop()
			return "", fmt.Errorf("scp-ing agent binary timed out: %v", scpAgentOutput.String())
		}
		return "", fmt.Errorf(
			"error copying agent binary to remote machine (%v): %v", err, scpAgentOutput.String())
	}

	// get the agent's revision after scp'ing over the executable
	postSCPAgentRevision, err := agbh.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting post scp agent "+
			"revision: %v", err)
	}

	if preSCPAgentRevision != postSCPAgentRevision {
		evergreen.Logger.Logf(slogger.WARN, "Agent revision was %v before scp "+
			"but is now %v. Using previous revision %v for host %v",
			preSCPAgentRevision, postSCPAgentRevision, preSCPAgentRevision,
			hostObj.Id)
	}

	return preSCPAgentRevision, nil
}