// terminate the passed-in slice of hosts. returns any errors that occur
// terminating the hosts
func terminateHosts(hosts []host.Host, settings *evergreen.Settings, reason string) []error {
errChan := make(chan error)
for _, h := range hosts {
evergreen.Logger.Logf(slogger.INFO, "Terminating host %v", h.Id)
// terminate the host in a goroutine, passing the host in as a parameter
// so that the variable isn't reused for subsequent iterations
go func(hostToTerminate host.Host) {
errChan <- func() error {
event.LogMonitorOperation(hostToTerminate.Id, reason)
err := util.RunFunctionWithTimeout(func() error {
return terminateHost(&hostToTerminate, settings)
}, 12*time.Minute)
if err != nil {
if err == util.ErrTimedOut {
return fmt.Errorf("timeout terminating host %v", hostToTerminate.Id)
}
return fmt.Errorf("error terminating host %v: %v", hostToTerminate.Id, err)
}
evergreen.Logger.Logf(slogger.INFO, "Successfully terminated host %v", hostToTerminate.Id)
return nil
}()
}(h)
}
var errors []error
for range hosts {
if err := <-errChan; err != nil {
errors = append(errors, err)
}
}
return errors
}
// copyScript writes a given script as file "name" to the target host. This works
// by creating a local copy of the script on the runner's machine, scping it over
// then removing the local copy.
func (init *HostInit) copyScript(target *host.Host, name, script string) error {
// parse the hostname into the user, host and port
hostInfo, err := util.ParseSSHInfo(target.Host)
if err != nil {
return err
}
user := target.Distro.User
if hostInfo.User != "" {
user = hostInfo.User
}
// create a temp file for the script
file, err := ioutil.TempFile("", name)
if err != nil {
return fmt.Errorf("error creating temporary script file: %v", err)
}
defer func() {
file.Close()
os.Remove(file.Name())
}()
expanded, err := init.expandScript(script)
if err != nil {
return fmt.Errorf("error expanding script for host %v: %v", target.Id, err)
}
if _, err := io.WriteString(file, expanded); err != nil {
return fmt.Errorf("error writing local script: %v", err)
}
cloudHost, err := providers.GetCloudHost(target, init.Settings)
if err != nil {
return fmt.Errorf("failed to get cloud host for %v: %v", target.Id, err)
}
sshOptions, err := cloudHost.GetSSHOptions()
if err != nil {
return fmt.Errorf("error getting ssh options for host %v: %v", target.Id, err)
}
var scpCmdStderr bytes.Buffer
scpCmd := &command.ScpCommand{
Source: file.Name(),
Dest: name,
Stdout: &scpCmdStderr,
Stderr: &scpCmdStderr,
RemoteHostName: hostInfo.Hostname,
User: user,
Options: append([]string{"-P", hostInfo.Port}, sshOptions...),
}
err = util.RunFunctionWithTimeout(scpCmd.Run, SCPTimeout)
if err != nil {
if err == util.ErrTimedOut {
scpCmd.Stop()
return fmt.Errorf("scp-ing script timed out")
}
return fmt.Errorf("error (%v) copying script to remote machine: %v",
err, scpCmdStderr.String())
}
return nil
}
// terminate the passed-in slice of hosts. returns any errors that occur
// terminating the hosts
func terminateHosts(hosts []host.Host, settings *evergreen.Settings) []error {
// used to store any errors that occur
var errors []error
// for terminating the different hosts in parallel
waitGroup := &sync.WaitGroup{}
// to ensure thread-safe appending to the errors
errsLock := &sync.Mutex{}
for _, h := range hosts {
evergreen.Logger.Logf(slogger.INFO, "Terminating host %v...", h.Id)
waitGroup.Add(1)
// terminate the host in a goroutine. pass the host in as a parameter
// so that the variable isn't reused for subsequent iterations
go func(hostToTerminate host.Host) {
defer waitGroup.Done()
// wrapper function to terminate the host
terminateFunc := func() error {
return terminateHost(&hostToTerminate, settings)
}
// run the function with a timeout
err := util.RunFunctionWithTimeout(terminateFunc, 10*time.Second)
if err == util.ErrTimedOut {
errsLock.Lock()
errors = append(errors, fmt.Errorf("timeout terminating"+
" host %v", hostToTerminate.Id))
errsLock.Unlock()
} else if err != nil {
errsLock.Lock()
errors = append(errors, fmt.Errorf("error terminating host:"+
" %v", err))
errsLock.Unlock()
} else {
evergreen.Logger.Logf(slogger.INFO, "Successfully terminated host"+
" %v", hostToTerminate.Id)
}
}(h)
}
// make sure all terminations finish
waitGroup.Wait()
return errors
}
// Start the agent process on the specified remote host, and have it run
// the specified task.
// Returns an error if starting the agent remotely fails.
func (self *AgentBasedHostGateway) startAgentOnRemote(
settings *evergreen.Settings, task *model.Task,
hostObj *host.Host, sshOptions []string) error {
// the path to the agent binary on the remote machine
pathToExecutable := filepath.Join(hostObj.Distro.WorkDir, "main")
// build the command to run on the remote machine
remoteCmd := fmt.Sprintf(
`%v -api_server "%v" -task_id "%v" -task_secret "%v" -log_prefix "%v" -https_cert "%v"`,
pathToExecutable, settings.ApiUrl, task.Id, task.Secret, filepath.Join(hostObj.Distro.WorkDir,
agentFile), settings.Expansions["api_httpscert_path"],
)
evergreen.Logger.Logf(slogger.INFO, "%v", remoteCmd)
// compute any info necessary to ssh into the host
hostInfo, err := util.ParseSSHInfo(hostObj.Host)
if err != nil {
return fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
}
// run the command to kick off the agent remotely
var startAgentLog bytes.Buffer
startAgentCmd := &command.RemoteCommand{
CmdString: remoteCmd,
Stdout: &startAgentLog,
Stderr: &startAgentLog,
RemoteHostName: hostInfo.Hostname,
User: hostObj.User,
Options: append([]string{"-p", hostInfo.Port}, sshOptions...),
Background: true,
}
// run the command to start the agent with a timeout
err = util.RunFunctionWithTimeout(
startAgentCmd.Run,
StartAgentTimeout,
)
if err != nil {
if err == util.ErrTimedOut {
startAgentCmd.Stop()
return fmt.Errorf("starting agent timed out")
}
return fmt.Errorf("error starting agent on host %v (%v): %v", hostObj.Id, err, startAgentLog)
}
return nil
}
// RunRemoteScript executes a shell script that already exists on the remote host,
// returning logs and any errors that occur. Logs may still be returned for some errors.
func RunRemoteScript(h *host.Host, script string, sshOptions []string) (string, error) {
// parse the hostname into the user, host and port
hostInfo, err := util.ParseSSHInfo(h.Host)
if err != nil {
return "", err
}
user := h.Distro.User
if hostInfo.User != "" {
user = hostInfo.User
}
// run the remote script as sudo, if appropriate
sudoStr := ""
if h.Distro.SetupAsSudo {
sudoStr = "sudo "
}
// run command to ssh into remote machine and execute script
sshCmdStd := &util.CappedWriter{
Buffer: &bytes.Buffer{},
MaxBytes: 1024 * 1024, // 1MB
}
cmd := &command.RemoteCommand{
CmdString: sudoStr + "sh " + script,
Stdout: sshCmdStd,
Stderr: sshCmdStd,
RemoteHostName: hostInfo.Hostname,
User: user,
Options: []string{"-p", hostInfo.Port},
Background: false,
}
// force creation of a tty if sudo
if h.Distro.SetupAsSudo {
cmd.Options = []string{"-t", "-t", "-p", hostInfo.Port}
}
cmd.Options = append(cmd.Options, sshOptions...)
// run the ssh command with given timeout
err = util.RunFunctionWithTimeout(
cmd.Run,
SSHTimeout,
)
return sshCmdStd.String(), err
}
func (init *HostInit) fetchRemoteTaskData(taskId, cliPath, confPath string, target *host.Host) error {
hostSSHInfo, err := util.ParseSSHInfo(target.Host)
if err != nil {
return fmt.Errorf("error parsing ssh info %v: %v", target.Host, err)
}
cloudHost, err := providers.GetCloudHost(target, init.Settings)
if err != nil {
return fmt.Errorf("Failed to get cloud host for %v: %v", target.Id, err)
}
sshOptions, err := cloudHost.GetSSHOptions()
if err != nil {
return fmt.Errorf("Error getting ssh options for host %v: %v", target.Id, err)
}
sshOptions = append(sshOptions, "-o", "UserKnownHostsFile=/dev/null")
/* TESTING ONLY
setupDebugSSHTunnel(path_to_ssh_keys, target.User, hostSSHInfo.Hostname)
*/
// When testing, use this writer to force a copy of the output to be written to standard out so
// that remote command failures also show up in server log output.
//cmdOutput := io.MultiWriter(&util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}, os.Stdout)
cmdOutput := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
makeShellCmd := &command.RemoteCommand{
CmdString: fmt.Sprintf("%s -c '%s' fetch -t %s --source --artifacts --dir='%s'", cliPath, confPath, taskId, target.Distro.WorkDir),
Stdout: cmdOutput,
Stderr: cmdOutput,
RemoteHostName: hostSSHInfo.Hostname,
User: target.User,
Options: append([]string{"-p", hostSSHInfo.Port}, sshOptions...),
}
// run the make shell command with a timeout
err = util.RunFunctionWithTimeout(makeShellCmd.Run, 15*time.Minute)
return err
}
// setupHost runs the specified setup script for an individual host. Returns
// the output from running the script remotely, as well as any error that
// occurs. If the script exits with a non-zero exit code, the error will be non-nil.
func (init *HostInit) setupHost(targetHost *host.Host) ([]byte, error) {
// fetch the appropriate cloud provider for the host
cloudMgr, err := providers.GetCloudManager(targetHost.Provider, init.Settings)
if err != nil {
return nil,
fmt.Errorf("failed to get cloud manager for host %v with provider %v: %v",
targetHost.Id, targetHost.Provider, err)
}
// mark the host as initializing
if err := targetHost.SetInitializing(); err != nil {
if err == mgo.ErrNotFound {
return nil, ErrHostAlreadyInitializing
} else {
return nil, fmt.Errorf("database error: %v", err)
}
}
// run the function scheduled for when the host is up
err = cloudMgr.OnUp(targetHost)
if err != nil {
// if this fails it is probably due to an API hiccup, so we keep going.
evergreen.Logger.Logf(slogger.WARN, "OnUp callback failed for host '%v': '%v'", targetHost.Id, err)
}
// run the remote setup script as sudo, if appropriate
sudoStr := ""
if targetHost.Distro.SetupAsSudo {
sudoStr = "sudo "
}
// parse the hostname into the user, host and port
hostInfo, err := util.ParseSSHInfo(targetHost.Host)
if err != nil {
return nil, err
}
user := targetHost.Distro.User
if hostInfo.User != "" {
user = hostInfo.User
}
// create a temp file for the setup script
fileName := "setup.sh"
file, err := ioutil.TempFile("", fileName)
if err != nil {
return nil, fmt.Errorf("error creating setup script: %v", err)
}
defer func() {
file.Close()
os.Remove(file.Name())
}()
// build the setup script
setup, err := init.buildSetupScript(targetHost)
if err != nil {
return nil, fmt.Errorf("error building setup script for host %v: %v", targetHost.Id, err)
}
// write the setup script to the file
if _, err := file.Write([]byte(setup)); err != nil {
return nil, fmt.Errorf("error writing remote setup script: %v", err)
}
cloudHost, err := providers.GetCloudHost(targetHost, init.Settings)
if err != nil {
return nil, fmt.Errorf("Failed to get cloud host for %v: %v", targetHost.Id, err)
}
sshOptions, err := cloudHost.GetSSHOptions()
if err != nil {
return nil, fmt.Errorf("Error getting ssh options for host %v: %v", targetHost.Id, err)
}
// copy setup script over to the remote machine
var scpSetupCmdStderr bytes.Buffer
scpSetupCmd := &command.ScpCommand{
Source: file.Name(),
Dest: fileName,
Stdout: &scpSetupCmdStderr,
Stderr: &scpSetupCmdStderr,
RemoteHostName: hostInfo.Hostname,
User: user,
Options: append([]string{"-P", hostInfo.Port}, sshOptions...),
}
// run the command to scp the setup script with a timeout
err = util.RunFunctionWithTimeout(
scpSetupCmd.Run,
SCPTimeout,
)
if err != nil {
if err == util.ErrTimedOut {
scpSetupCmd.Stop()
return nil, fmt.Errorf("scp-ing setup script timed out")
}
return nil, fmt.Errorf("error (%v) copying setup script to remote "+
"machine: %v", err, scpSetupCmdStderr.String())
}
// run command to ssh into remote machine and execute setup script
var sshSetupCmdStderr bytes.Buffer
runSetupCmd := &command.RemoteCommand{
CmdString: sudoStr + "sh " + fileName,
Stdout: &sshSetupCmdStderr,
Stderr: &sshSetupCmdStderr,
RemoteHostName: hostInfo.Hostname,
User: user,
Options: []string{"-p", hostInfo.Port},
Background: false,
}
// only force creation of a tty if sudo
if targetHost.Distro.SetupAsSudo {
runSetupCmd.Options = []string{"-t", "-t", "-p", hostInfo.Port}
}
runSetupCmd.Options = append(runSetupCmd.Options, sshOptions...)
// run the ssh command with given timeout
err = util.RunFunctionWithTimeout(
runSetupCmd.Run,
time.Duration(SSHTimeoutSeconds)*time.Second,
)
return sshSetupCmdStderr.Bytes(), err
}
// LoadClient places the evergreen command line client on the host, places a copy of the user's
// settings onto the host, and makes the binary appear in the $PATH when the user logs in.
// If successful, returns an instance of LoadClientResult which contains the paths where the
// binary and config file were written to.
func (init *HostInit) LoadClient(target *host.Host) (*LoadClientResult, error) {
// Make sure we have the binary we want to upload - if it hasn't been built for the given
// architecture, fail early
cliBinaryPath, err := LocateCLIBinary(init.Settings, target.Distro.Arch)
if err != nil {
return nil, fmt.Errorf("couldn't locate CLI binary for upload: %v", err)
}
if target.ProvisionOptions == nil {
return nil, fmt.Errorf("ProvisionOptions is nil")
}
if target.ProvisionOptions.OwnerId == "" {
return nil, fmt.Errorf("OwnerId not set")
}
// get the information about the owner of the host
owner, err := user.FindOne(user.ById(target.ProvisionOptions.OwnerId))
if err != nil {
return nil, fmt.Errorf("couldn't fetch owner %v for host: %v", target.ProvisionOptions.OwnerId, err)
}
// 1. mkdir the destination directory on the host,
// and modify ~/.profile so the target binary will be on the $PATH
targetDir := "cli_bin"
hostSSHInfo, err := util.ParseSSHInfo(target.Host)
if err != nil {
return nil, fmt.Errorf("error parsing ssh info %v: %v", target.Host, err)
}
cloudHost, err := providers.GetCloudHost(target, init.Settings)
if err != nil {
return nil, fmt.Errorf("Failed to get cloud host for %v: %v", target.Id, err)
}
sshOptions, err := cloudHost.GetSSHOptions()
if err != nil {
return nil, fmt.Errorf("Error getting ssh options for host %v: %v", target.Id, err)
}
sshOptions = append(sshOptions, "-o", "UserKnownHostsFile=/dev/null")
mkdirOutput := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
// Create the directory for the binary to be uploaded into.
// Also, make a best effort to add the binary's location to $PATH upon login. If we can't do
// this successfully, the command will still succeed, it just means that the user will have to
// use an absolute path (or manually set $PATH in their shell) to execute it.
makeShellCmd := &command.RemoteCommand{
CmdString: fmt.Sprintf("mkdir -m 777 -p ~/%s && (echo 'PATH=$PATH:~/%s' >> ~/.profile || true; echo 'PATH=$PATH:~/%s' >> ~/.bash_profile || true)", targetDir, targetDir, targetDir),
Stdout: mkdirOutput,
Stderr: mkdirOutput,
RemoteHostName: hostSSHInfo.Hostname,
User: target.User,
Options: append([]string{"-p", hostSSHInfo.Port}, sshOptions...),
}
scpOut := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
// run the make shell command with a timeout
err = util.RunFunctionWithTimeout(makeShellCmd.Run, 30*time.Second)
if err != nil {
return nil, fmt.Errorf("error running setup command for cli, %v: '%v'", mkdirOutput.Buffer.String(), err)
}
// place the binary into the directory
scpSetupCmd := &command.ScpCommand{
Source: cliBinaryPath,
Dest: fmt.Sprintf("~/%s/evergreen", targetDir),
Stdout: scpOut,
Stderr: scpOut,
RemoteHostName: hostSSHInfo.Hostname,
User: target.User,
Options: append([]string{"-P", hostSSHInfo.Port}, sshOptions...),
}
// run the command to scp the setup script with a timeout
err = util.RunFunctionWithTimeout(scpSetupCmd.Run, 3*time.Minute)
if err != nil {
return nil, fmt.Errorf("error running SCP command for cli, %v: '%v'", scpOut.Buffer.String(), err)
}
// 4. Write a settings file for the user that owns the host, and scp it to the directory
outputStruct := model.CLISettings{
User: owner.Id,
APIKey: owner.APIKey,
APIServerHost: init.Settings.ApiUrl + "/api",
UIServerHost: init.Settings.Ui.Url,
}
outputJSON, err := json.Marshal(outputStruct)
if err != nil {
return nil, err
}
tempFileName, err := util.WriteTempFile("", outputJSON)
if err != nil {
return nil, err
}
defer os.Remove(tempFileName)
scpYmlCommand := &command.ScpCommand{
Source: tempFileName,
Dest: fmt.Sprintf("~/%s/.evergreen.yml", targetDir),
Stdout: scpOut,
Stderr: scpOut,
RemoteHostName: hostSSHInfo.Hostname,
User: target.User,
Options: append([]string{"-P", hostSSHInfo.Port}, sshOptions...),
}
err = util.RunFunctionWithTimeout(scpYmlCommand.Run, 30*time.Second)
if err != nil {
return nil, fmt.Errorf("error running SCP command for evergreen.yml, %v: '%v'", scpOut.Buffer.String(), err)
}
return &LoadClientResult{
BinaryPath: fmt.Sprintf("~/%s/evergreen", targetDir),
ConfigPath: fmt.Sprintf("~/%s/.evergreen.yml", targetDir),
}, nil
}
// Prepare the remote machine to run a task.
func (self *AgentBasedHostGateway) prepRemoteHost(settings *evergreen.Settings,
hostObj host.Host, sshOptions []string, mciHome string) (string, error) {
// compute any info necessary to ssh into the host
hostInfo, err := util.ParseSSHInfo(hostObj.Host)
if err != nil {
return "", fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
}
// first, create the necessary sandbox of directories on the remote machine
makeShellCmd := &command.RemoteCommand{
CmdString: fmt.Sprintf("mkdir -m 777 -p %v", hostObj.Distro.WorkDir),
Stdout: ioutil.Discard, // TODO(EVG-233) change to real logging
Stderr: ioutil.Discard,
RemoteHostName: hostInfo.Hostname,
User: hostObj.User,
Options: append([]string{"-p", hostInfo.Port}, sshOptions...),
Background: false,
}
evergreen.Logger.Logf(slogger.INFO, "Directories command: '%#v'", makeShellCmd)
// run the make shell command with a timeout
err = util.RunFunctionWithTimeout(
makeShellCmd.Run,
MakeShellTimeout,
)
if err != nil {
// if it timed out, kill the command
if err == util.ErrTimedOut {
makeShellCmd.Stop()
return "", fmt.Errorf("creating remote directories timed out")
}
return "", fmt.Errorf("error creating directories on remote machine: %v", err)
}
scpConfigsCmd := &command.ScpCommand{
Source: filepath.Join(mciHome, settings.ConfigDir),
Dest: hostObj.Distro.WorkDir,
Stdout: ioutil.Discard, // TODO(EVG-233) change to real logging
Stderr: ioutil.Discard,
RemoteHostName: hostInfo.Hostname,
User: hostObj.User,
Options: append([]string{"-P", hostInfo.Port, "-r"},
sshOptions...),
}
// run the command to scp the configs with a timeout
err = util.RunFunctionWithTimeout(
scpConfigsCmd.Run,
SCPTimeout,
)
if err != nil {
// if it timed out, kill the scp command
if err == util.ErrTimedOut {
scpConfigsCmd.Stop()
return "", fmt.Errorf("scp-ing config directory timed out")
}
return "", fmt.Errorf("error copying config directory to remote: "+
"machine %v", err)
}
// third, copy over the correct agent binary to the remote machine
var scpAgentCmdStderr bytes.Buffer
execSubPath, err := executableSubPath(hostObj.Distro.Id)
if err != nil {
return "", fmt.Errorf("error computing subpath to executable: %v", err)
}
scpAgentCmd := &command.ScpCommand{
Source: filepath.Join(self.ExecutablesDir, execSubPath),
Dest: hostObj.Distro.WorkDir,
Stdout: ioutil.Discard, // TODO(EVG-233) change to real logging
Stderr: &scpAgentCmdStderr,
RemoteHostName: hostInfo.Hostname,
User: hostObj.User,
Options: append([]string{"-P", hostInfo.Port}, sshOptions...),
}
// get the agent's revision before scp'ing over the executable
preSCPAgentRevision, err := self.GetAgentRevision()
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error getting pre scp agent "+
"revision: %v", err)
}
// run the command to scp the agent with a timeout
err = util.RunFunctionWithTimeout(
scpAgentCmd.Run,
SCPTimeout,
)
if err != nil {
if err == util.ErrTimedOut {
scpAgentCmd.Stop()
return "", fmt.Errorf("scp-ing agent binary timed out")
}
return "", fmt.Errorf("error (%v) copying agent binary to remote "+
"machine: %v", err, scpAgentCmdStderr.String())
}
// get the agent's revision after scp'ing over the executable
postSCPAgentRevision, err := self.GetAgentRevision()
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error getting post scp agent "+
"revision: %v", err)
}
if preSCPAgentRevision != postSCPAgentRevision {
evergreen.Logger.Logf(slogger.WARN, "Agent revision was %v before scp "+
"but is now %v. Using previous revision %v for host %v",
preSCPAgentRevision, postSCPAgentRevision, preSCPAgentRevision,
hostObj.Id)
}
return preSCPAgentRevision, nil
}
// Prepare the remote machine to run a task.
// This consists of:
// 1. Creating the directories on the remote host where, according to the distro's settings,
// the agent should be placed.
// 2. Copying the agent into that directory.
func (agbh *AgentHostGateway) prepRemoteHost(hostObj host.Host, sshOptions []string) (string, error) {
// compute any info necessary to ssh into the host
hostInfo, err := util.ParseSSHInfo(hostObj.Host)
if err != nil {
return "", fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
}
// first, create the necessary sandbox of directories on the remote machine
mkdirOutput := newCappedOutputLog()
makeShellCmd := &command.RemoteCommand{
Id: fmt.Sprintf("agent_mkdir-%v", rand.Int()),
CmdString: fmt.Sprintf("mkdir -m 777 -p %v", hostObj.Distro.WorkDir),
Stdout: mkdirOutput,
Stderr: mkdirOutput,
RemoteHostName: hostInfo.Hostname,
User: hostObj.User,
Options: append([]string{"-p", hostInfo.Port}, sshOptions...),
Background: false,
}
evergreen.Logger.Logf(slogger.INFO, "Directories command: '%#v'", makeShellCmd)
// run the make shell command with a timeout
err = util.RunFunctionWithTimeout(makeShellCmd.Run, MakeShellTimeout)
if err != nil {
// if it timed out, kill the command
if err == util.ErrTimedOut {
makeShellCmd.Stop()
return "", fmt.Errorf("creating remote directories timed out: %v", mkdirOutput.String())
}
return "", fmt.Errorf(
"error creating directories on remote machine (%v): %v", err, mkdirOutput.String())
}
// third, copy over the correct agent binary to the remote machine
execSubPath, err := executableSubPath(hostObj.Distro.Id)
if err != nil {
return "", fmt.Errorf("error computing subpath to executable: %v", err)
}
scpAgentOutput := newCappedOutputLog()
scpAgentCmd := &command.ScpCommand{
Id: fmt.Sprintf("scp%v", rand.Int()),
Source: filepath.Join(agbh.ExecutablesDir, execSubPath),
Dest: hostObj.Distro.WorkDir,
Stdout: scpAgentOutput,
Stderr: scpAgentOutput,
RemoteHostName: hostInfo.Hostname,
User: hostObj.User,
Options: append([]string{"-P", hostInfo.Port}, sshOptions...),
}
// get the agent's revision before scp'ing over the executable
preSCPAgentRevision, err := agbh.GetAgentRevision()
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error getting pre scp agent "+
"revision: %v", err)
}
// run the command to scp the agent with a timeout
err = util.RunFunctionWithTimeout(scpAgentCmd.Run, SCPTimeout)
if err != nil {
if err == util.ErrTimedOut {
scpAgentCmd.Stop()
return "", fmt.Errorf("scp-ing agent binary timed out: %v", scpAgentOutput.String())
}
return "", fmt.Errorf(
"error copying agent binary to remote machine (%v): %v", err, scpAgentOutput.String())
}
// get the agent's revision after scp'ing over the executable
postSCPAgentRevision, err := agbh.GetAgentRevision()
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error getting post scp agent "+
"revision: %v", err)
}
if preSCPAgentRevision != postSCPAgentRevision {
evergreen.Logger.Logf(slogger.WARN, "Agent revision was %v before scp "+
"but is now %v. Using previous revision %v for host %v",
preSCPAgentRevision, postSCPAgentRevision, preSCPAgentRevision,
hostObj.Id)
}
return preSCPAgentRevision, nil
}