Beispiel #1
0
// copyScript writes a given script as file "name" to the target host. This works
// by creating a local copy of the script on the runner's machine, scping it over
// then removing the local copy.
func (init *HostInit) copyScript(target *host.Host, name, script string) error {
	// parse the hostname into the user, host and port
	hostInfo, err := util.ParseSSHInfo(target.Host)
	if err != nil {
		return err
	}
	user := target.Distro.User
	if hostInfo.User != "" {
		user = hostInfo.User
	}

	// create a temp file for the script
	file, err := ioutil.TempFile("", name)
	if err != nil {
		return fmt.Errorf("error creating temporary script file: %v", err)
	}
	defer func() {
		file.Close()
		os.Remove(file.Name())
	}()

	expanded, err := init.expandScript(script)
	if err != nil {
		return fmt.Errorf("error expanding script for host %v: %v", target.Id, err)
	}
	if _, err := io.WriteString(file, expanded); err != nil {
		return fmt.Errorf("error writing local script: %v", err)
	}

	cloudHost, err := providers.GetCloudHost(target, init.Settings)
	if err != nil {
		return fmt.Errorf("failed to get cloud host for %v: %v", target.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return fmt.Errorf("error getting ssh options for host %v: %v", target.Id, err)
	}

	var scpCmdStderr bytes.Buffer
	scpCmd := &command.ScpCommand{
		Source:         file.Name(),
		Dest:           name,
		Stdout:         &scpCmdStderr,
		Stderr:         &scpCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}
	err = util.RunFunctionWithTimeout(scpCmd.Run, SCPTimeout)
	if err != nil {
		if err == util.ErrTimedOut {
			scpCmd.Stop()
			return fmt.Errorf("scp-ing script timed out")
		}
		return fmt.Errorf("error (%v) copying script to remote machine: %v",
			err, scpCmdStderr.String())
	}
	return nil
}
Beispiel #2
0
func TestHostSshParse(t *testing.T) {
	Convey("Parsing a string with hostname and port should be parsed correctly", t, func() {
		hostname1 := "host:123"
		host1, err := util.ParseSSHInfo(hostname1)
		So(err, ShouldBeNil)
		So(host1.User, ShouldEqual, "")
		So(host1.Hostname, ShouldEqual, "host")
		So(host1.Port, ShouldEqual, "123")
	})

	Convey("Parsing a string with only a hostname should assume default port", t, func() {
		hostname2 := "host"
		host2, err := util.ParseSSHInfo(hostname2)
		So(err, ShouldBeNil)
		So(host2.User, ShouldEqual, "")
		So(host2.Hostname, ShouldEqual, "host")
		So(host2.Port, ShouldEqual, "22")
	})

	Convey("Parsing a string with user, hostname, and port should extract all 3", t, func() {
		hostname3 := "user@host:400"
		host3, err := util.ParseSSHInfo(hostname3)
		So(err, ShouldBeNil)
		So(host3.User, ShouldEqual, "user")
		So(host3.Hostname, ShouldEqual, "host")
		So(host3.Port, ShouldEqual, "400")
	})

	Convey("Parsing a string with user and host should assume the default port", t, func() {
		hostname4 := "user@host"
		host4, err := util.ParseSSHInfo(hostname4)
		So(err, ShouldBeNil)
		So(host4.User, ShouldEqual, "user")
		So(host4.Hostname, ShouldEqual, "host")
		So(host4.Port, ShouldEqual, "22")
	})
}
Beispiel #3
0
// Start the agent process on the specified remote host, and have it run
// the specified task.
// Returns an error if starting the agent remotely fails.
func (self *AgentBasedHostGateway) startAgentOnRemote(
	settings *evergreen.Settings, task *model.Task,
	hostObj *host.Host, sshOptions []string) error {

	// the path to the agent binary on the remote machine
	pathToExecutable := filepath.Join(hostObj.Distro.WorkDir, "main")

	// build the command to run on the remote machine
	remoteCmd := fmt.Sprintf(
		`%v -api_server "%v" -task_id "%v" -task_secret "%v" -log_prefix "%v" -https_cert "%v"`,
		pathToExecutable, settings.ApiUrl, task.Id, task.Secret, filepath.Join(hostObj.Distro.WorkDir,
			agentFile), settings.Expansions["api_httpscert_path"],
	)
	evergreen.Logger.Logf(slogger.INFO, "%v", remoteCmd)

	// compute any info necessary to ssh into the host
	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
	}

	// run the command to kick off the agent remotely
	var startAgentLog bytes.Buffer
	startAgentCmd := &command.RemoteCommand{
		CmdString:      remoteCmd,
		Stdout:         &startAgentLog,
		Stderr:         &startAgentLog,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     true,
	}

	// run the command to start the agent with a timeout
	err = util.RunFunctionWithTimeout(
		startAgentCmd.Run,
		StartAgentTimeout,
	)
	if err != nil {
		if err == util.ErrTimedOut {
			startAgentCmd.Stop()
			return fmt.Errorf("starting agent timed out")
		}
		return fmt.Errorf("error starting agent on host %v (%v): %v", hostObj.Id, err, startAgentLog)
	}

	return nil
}
func UpdateStaticHosts(e *evergreen.Settings) error {
	distros, err := distro.Find(distro.ByProvider(static.ProviderName))
	if err != nil {
		return err
	}
	activeStaticHosts := make([]string, 0)
	settings := &static.Settings{}

	for _, d := range distros {
		err = mapstructure.Decode(d.ProviderSettings, settings)
		if err != nil {
			return fmt.Errorf("invalid static settings for '%v'", d.Id)
		}
		for _, h := range settings.Hosts {
			hostInfo, err := util.ParseSSHInfo(h.Name)
			if err != nil {
				return err
			}
			user := hostInfo.User
			if user == "" {
				user = d.User
			}
			staticHost := host.Host{
				Id:           h.Name,
				User:         user,
				Host:         h.Name,
				Distro:       d,
				CreationTime: time.Now(),
				Provider:     evergreen.HostTypeStatic,
				StartedBy:    evergreen.User,
				Status:       evergreen.HostRunning,
				Provisioned:  true,
			}

			// upsert the host
			_, err = staticHost.Upsert()
			if err != nil {
				return err
			}
			activeStaticHosts = append(activeStaticHosts, h.Name)
		}
	}
	return host.DecommissionInactiveStaticHosts(activeStaticHosts)
}
Beispiel #5
0
// RunRemoteScript executes a shell script that already exists on the remote host,
// returning logs and any errors that occur. Logs may still be returned for some errors.
func RunRemoteScript(h *host.Host, script string, sshOptions []string) (string, error) {
	// parse the hostname into the user, host and port
	hostInfo, err := util.ParseSSHInfo(h.Host)
	if err != nil {
		return "", err
	}
	user := h.Distro.User
	if hostInfo.User != "" {
		user = hostInfo.User
	}

	// run the remote script as sudo, if appropriate
	sudoStr := ""
	if h.Distro.SetupAsSudo {
		sudoStr = "sudo "
	}
	// run command to ssh into remote machine and execute script
	sshCmdStd := &util.CappedWriter{
		Buffer:   &bytes.Buffer{},
		MaxBytes: 1024 * 1024, // 1MB
	}
	cmd := &command.RemoteCommand{
		CmdString:      sudoStr + "sh " + script,
		Stdout:         sshCmdStd,
		Stderr:         sshCmdStd,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        []string{"-p", hostInfo.Port},
		Background:     false,
	}
	// force creation of a tty if sudo
	if h.Distro.SetupAsSudo {
		cmd.Options = []string{"-t", "-t", "-p", hostInfo.Port}
	}
	cmd.Options = append(cmd.Options, sshOptions...)

	// run the ssh command with given timeout
	err = util.RunFunctionWithTimeout(
		cmd.Run,
		SSHTimeout,
	)
	return sshCmdStd.String(), err
}
Beispiel #6
0
//CheckSSHResponse runs a test command over SSH to check whether or not the host
//appears to be up and accepting ssh connections. Returns true/false if the check
//passes or fails, or an error if the command cannot be attempted.
func CheckSSHResponse(hostObject *host.Host, sshOptions []string) (bool, error) {
	hostInfo, err := util.ParseSSHInfo(hostObject.Host)
	if err != nil {
		return false, err
	}

	if hostInfo.User == "" {
		hostInfo.User = hostObject.User
	}

	// construct a command to check reachability
	remoteCommand := &command.RemoteCommand{
		CmdString:      "echo hi",
		Stdout:         ioutil.Discard,
		Stderr:         ioutil.Discard,
		RemoteHostName: hostInfo.Hostname,
		User:           hostInfo.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     false,
	}

	done := make(chan error)
	err = remoteCommand.Start()
	if err != nil {
		return false, err
	}

	go func() {
		done <- remoteCommand.Wait()
	}()

	select {
	case <-time.After(HostCheckTimeout):
		remoteCommand.Stop()
		return false, nil
	case err = <-done:
		if err != nil {
			return false, nil
		}
		return true, nil
	}
}
Beispiel #7
0
func (init *HostInit) fetchRemoteTaskData(taskId, cliPath, confPath string, target *host.Host) error {
	hostSSHInfo, err := util.ParseSSHInfo(target.Host)
	if err != nil {
		return fmt.Errorf("error parsing ssh info %v: %v", target.Host, err)
	}

	cloudHost, err := providers.GetCloudHost(target, init.Settings)
	if err != nil {
		return fmt.Errorf("Failed to get cloud host for %v: %v", target.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return fmt.Errorf("Error getting ssh options for host %v: %v", target.Id, err)
	}
	sshOptions = append(sshOptions, "-o", "UserKnownHostsFile=/dev/null")

	/* TESTING ONLY
	setupDebugSSHTunnel(path_to_ssh_keys, target.User, hostSSHInfo.Hostname)
	*/

	// When testing, use this writer to force a copy of the output to be written to standard out so
	// that remote command failures also show up in server log output.
	//cmdOutput := io.MultiWriter(&util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}, os.Stdout)

	cmdOutput := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
	makeShellCmd := &command.RemoteCommand{
		CmdString:      fmt.Sprintf("%s -c '%s' fetch -t %s --source --artifacts --dir='%s'", cliPath, confPath, taskId, target.Distro.WorkDir),
		Stdout:         cmdOutput,
		Stderr:         cmdOutput,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-p", hostSSHInfo.Port}, sshOptions...),
	}

	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(makeShellCmd.Run, 15*time.Minute)
	return err
}
Beispiel #8
0
// constructPwdUpdateCommand returns a RemoteCommand struct used to
// set the RDP password on a remote windows machine.
func constructPwdUpdateCommand(settings *evergreen.Settings, hostObj *host.Host,
	password string) (*command.RemoteCommand, error) {

	cloudHost, err := providers.GetCloudHost(hostObj, settings)
	if err != nil {
		return nil, err
	}

	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return nil, err
	}

	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return nil, err
	}

	outputLineHandler := evergreen.NewInfoLoggingWriter(&evergreen.Logger)
	errorLineHandler := evergreen.NewErrorLoggingWriter(&evergreen.Logger)

	updatePwdCmd := fmt.Sprintf("net user %v %v && sc config "+
		"sshd obj= '.\\%v' password= \"%v\"", hostObj.User, password,
		hostObj.User, password)

	// construct the required termination command
	remoteCommand := &command.RemoteCommand{
		CmdString:       updatePwdCmd,
		Stdout:          outputLineHandler,
		Stderr:          errorLineHandler,
		LoggingDisabled: true,
		RemoteHostName:  hostInfo.Hostname,
		User:            hostObj.User,
		Options:         append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:      false,
	}
	return remoteCommand, nil
}
Beispiel #9
0
// setupHost runs the specified setup script for an individual host. Returns
// the output from running the script remotely, as well as any error that
// occurs. If the script exits with a non-zero exit code, the error will be non-nil.
func (init *HostInit) setupHost(targetHost *host.Host) ([]byte, error) {

	// fetch the appropriate cloud provider for the host
	cloudMgr, err := providers.GetCloudManager(targetHost.Provider, init.Settings)
	if err != nil {
		return nil,
			fmt.Errorf("failed to get cloud manager for host %v with provider %v: %v",
				targetHost.Id, targetHost.Provider, err)
	}

	// mark the host as initializing
	if err := targetHost.SetInitializing(); err != nil {
		if err == mgo.ErrNotFound {
			return nil, ErrHostAlreadyInitializing
		} else {
			return nil, fmt.Errorf("database error: %v", err)
		}
	}

	// run the function scheduled for when the host is up
	err = cloudMgr.OnUp(targetHost)
	if err != nil {
		// if this fails it is probably due to an API hiccup, so we keep going.
		evergreen.Logger.Logf(slogger.WARN, "OnUp callback failed for host '%v': '%v'", targetHost.Id, err)
	}

	// run the remote setup script as sudo, if appropriate
	sudoStr := ""
	if targetHost.Distro.SetupAsSudo {
		sudoStr = "sudo "
	}

	// parse the hostname into the user, host and port
	hostInfo, err := util.ParseSSHInfo(targetHost.Host)
	if err != nil {
		return nil, err
	}
	user := targetHost.Distro.User
	if hostInfo.User != "" {
		user = hostInfo.User
	}

	// create a temp file for the setup script
	fileName := "setup.sh"
	file, err := ioutil.TempFile("", fileName)
	if err != nil {
		return nil, fmt.Errorf("error creating setup script: %v", err)
	}
	defer func() {
		file.Close()
		os.Remove(file.Name())
	}()

	// build the setup script
	setup, err := init.buildSetupScript(targetHost)
	if err != nil {
		return nil, fmt.Errorf("error building setup script for host %v: %v", targetHost.Id, err)
	}

	// write the setup script to the file
	if _, err := file.Write([]byte(setup)); err != nil {
		return nil, fmt.Errorf("error writing remote setup script: %v", err)
	}

	cloudHost, err := providers.GetCloudHost(targetHost, init.Settings)
	if err != nil {
		return nil, fmt.Errorf("Failed to get cloud host for %v: %v", targetHost.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return nil, fmt.Errorf("Error getting ssh options for host %v: %v", targetHost.Id, err)
	}

	// copy setup script over to the remote machine
	var scpSetupCmdStderr bytes.Buffer
	scpSetupCmd := &command.ScpCommand{
		Source:         file.Name(),
		Dest:           fileName,
		Stdout:         &scpSetupCmdStderr,
		Stderr:         &scpSetupCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}

	// run the command to scp the setup script with a timeout
	err = util.RunFunctionWithTimeout(
		scpSetupCmd.Run,
		SCPTimeout,
	)
	if err != nil {
		if err == util.ErrTimedOut {
			scpSetupCmd.Stop()
			return nil, fmt.Errorf("scp-ing setup script timed out")
		}
		return nil, fmt.Errorf("error (%v) copying setup script to remote "+
			"machine: %v", err, scpSetupCmdStderr.String())
	}

	// run command to ssh into remote machine and execute setup script
	var sshSetupCmdStderr bytes.Buffer
	runSetupCmd := &command.RemoteCommand{
		CmdString:      sudoStr + "sh " + fileName,
		Stdout:         &sshSetupCmdStderr,
		Stderr:         &sshSetupCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           user,
		Options:        []string{"-p", hostInfo.Port},
		Background:     false,
	}

	// only force creation of a tty if sudo
	if targetHost.Distro.SetupAsSudo {
		runSetupCmd.Options = []string{"-t", "-t", "-p", hostInfo.Port}
	}
	runSetupCmd.Options = append(runSetupCmd.Options, sshOptions...)

	// run the ssh command with given timeout
	err = util.RunFunctionWithTimeout(
		runSetupCmd.Run,
		time.Duration(SSHTimeoutSeconds)*time.Second,
	)

	return sshSetupCmdStderr.Bytes(), err
}
Beispiel #10
0
// LoadClient places the evergreen command line client on the host, places a copy of the user's
// settings onto the host, and makes the binary appear in the $PATH when the user logs in.
// If successful, returns an instance of LoadClientResult which contains the paths where the
// binary and config file were written to.
func (init *HostInit) LoadClient(target *host.Host) (*LoadClientResult, error) {
	// Make sure we have the binary we want to upload - if it hasn't been built for the given
	// architecture, fail early
	cliBinaryPath, err := LocateCLIBinary(init.Settings, target.Distro.Arch)
	if err != nil {
		return nil, fmt.Errorf("couldn't locate CLI binary for upload: %v", err)
	}
	if target.ProvisionOptions == nil {
		return nil, fmt.Errorf("ProvisionOptions is nil")
	}
	if target.ProvisionOptions.OwnerId == "" {
		return nil, fmt.Errorf("OwnerId not set")
	}

	// get the information about the owner of the host
	owner, err := user.FindOne(user.ById(target.ProvisionOptions.OwnerId))
	if err != nil {
		return nil, fmt.Errorf("couldn't fetch owner %v for host: %v", target.ProvisionOptions.OwnerId, err)
	}

	// 1. mkdir the destination directory on the host,
	//    and modify ~/.profile so the target binary will be on the $PATH
	targetDir := "cli_bin"
	hostSSHInfo, err := util.ParseSSHInfo(target.Host)
	if err != nil {
		return nil, fmt.Errorf("error parsing ssh info %v: %v", target.Host, err)
	}

	cloudHost, err := providers.GetCloudHost(target, init.Settings)
	if err != nil {
		return nil, fmt.Errorf("Failed to get cloud host for %v: %v", target.Id, err)
	}
	sshOptions, err := cloudHost.GetSSHOptions()
	if err != nil {
		return nil, fmt.Errorf("Error getting ssh options for host %v: %v", target.Id, err)
	}
	sshOptions = append(sshOptions, "-o", "UserKnownHostsFile=/dev/null")

	mkdirOutput := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}

	// Create the directory for the binary to be uploaded into.
	// Also, make a best effort to add the binary's location to $PATH upon login. If we can't do
	// this successfully, the command will still succeed, it just means that the user will have to
	// use an absolute path (or manually set $PATH in their shell) to execute it.
	makeShellCmd := &command.RemoteCommand{
		CmdString:      fmt.Sprintf("mkdir -m 777 -p ~/%s && (echo 'PATH=$PATH:~/%s' >> ~/.profile || true; echo 'PATH=$PATH:~/%s' >> ~/.bash_profile || true)", targetDir, targetDir, targetDir),
		Stdout:         mkdirOutput,
		Stderr:         mkdirOutput,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-p", hostSSHInfo.Port}, sshOptions...),
	}

	scpOut := &util.CappedWriter{&bytes.Buffer{}, 1024 * 1024}
	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(makeShellCmd.Run, 30*time.Second)
	if err != nil {
		return nil, fmt.Errorf("error running setup command for cli, %v: '%v'", mkdirOutput.Buffer.String(), err)
	}
	// place the binary into the directory
	scpSetupCmd := &command.ScpCommand{
		Source:         cliBinaryPath,
		Dest:           fmt.Sprintf("~/%s/evergreen", targetDir),
		Stdout:         scpOut,
		Stderr:         scpOut,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-P", hostSSHInfo.Port}, sshOptions...),
	}

	// run the command to scp the setup script with a timeout
	err = util.RunFunctionWithTimeout(scpSetupCmd.Run, 3*time.Minute)
	if err != nil {
		return nil, fmt.Errorf("error running SCP command for cli, %v: '%v'", scpOut.Buffer.String(), err)
	}

	// 4. Write a settings file for the user that owns the host, and scp it to the directory
	outputStruct := model.CLISettings{
		User:          owner.Id,
		APIKey:        owner.APIKey,
		APIServerHost: init.Settings.ApiUrl + "/api",
		UIServerHost:  init.Settings.Ui.Url,
	}
	outputJSON, err := json.Marshal(outputStruct)
	if err != nil {
		return nil, err
	}

	tempFileName, err := util.WriteTempFile("", outputJSON)
	if err != nil {
		return nil, err
	}
	defer os.Remove(tempFileName)

	scpYmlCommand := &command.ScpCommand{
		Source:         tempFileName,
		Dest:           fmt.Sprintf("~/%s/.evergreen.yml", targetDir),
		Stdout:         scpOut,
		Stderr:         scpOut,
		RemoteHostName: hostSSHInfo.Hostname,
		User:           target.User,
		Options:        append([]string{"-P", hostSSHInfo.Port}, sshOptions...),
	}
	err = util.RunFunctionWithTimeout(scpYmlCommand.Run, 30*time.Second)
	if err != nil {
		return nil, fmt.Errorf("error running SCP command for evergreen.yml, %v: '%v'", scpOut.Buffer.String(), err)
	}

	return &LoadClientResult{
		BinaryPath: fmt.Sprintf("~/%s/evergreen", targetDir),
		ConfigPath: fmt.Sprintf("~/%s/.evergreen.yml", targetDir),
	}, nil
}
Beispiel #11
0
// Prepare the remote machine to run a task.
func (self *AgentBasedHostGateway) prepRemoteHost(settings *evergreen.Settings,
	hostObj host.Host, sshOptions []string, mciHome string) (string, error) {

	// compute any info necessary to ssh into the host
	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return "", fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
	}

	// first, create the necessary sandbox of directories on the remote machine
	makeShellCmd := &command.RemoteCommand{
		CmdString:      fmt.Sprintf("mkdir -m 777 -p %v", hostObj.Distro.WorkDir),
		Stdout:         ioutil.Discard, // TODO(EVG-233) change to real logging
		Stderr:         ioutil.Discard,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     false,
	}

	evergreen.Logger.Logf(slogger.INFO, "Directories command: '%#v'", makeShellCmd)

	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(
		makeShellCmd.Run,
		MakeShellTimeout,
	)
	if err != nil {
		// if it timed out, kill the command
		if err == util.ErrTimedOut {
			makeShellCmd.Stop()
			return "", fmt.Errorf("creating remote directories timed out")
		}
		return "", fmt.Errorf("error creating directories on remote machine: %v", err)
	}

	scpConfigsCmd := &command.ScpCommand{
		Source:         filepath.Join(mciHome, settings.ConfigDir),
		Dest:           hostObj.Distro.WorkDir,
		Stdout:         ioutil.Discard, // TODO(EVG-233) change to real logging
		Stderr:         ioutil.Discard,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options: append([]string{"-P", hostInfo.Port, "-r"},
			sshOptions...),
	}

	// run the command to scp the configs with a timeout
	err = util.RunFunctionWithTimeout(
		scpConfigsCmd.Run,
		SCPTimeout,
	)
	if err != nil {
		// if it timed out, kill the scp command
		if err == util.ErrTimedOut {
			scpConfigsCmd.Stop()
			return "", fmt.Errorf("scp-ing config directory timed out")
		}
		return "", fmt.Errorf("error copying config directory to remote: "+
			"machine %v", err)
	}

	// third, copy over the correct agent binary to the remote machine
	var scpAgentCmdStderr bytes.Buffer
	execSubPath, err := executableSubPath(hostObj.Distro.Id)
	if err != nil {
		return "", fmt.Errorf("error computing subpath to executable: %v", err)
	}
	scpAgentCmd := &command.ScpCommand{
		Source:         filepath.Join(self.ExecutablesDir, execSubPath),
		Dest:           hostObj.Distro.WorkDir,
		Stdout:         ioutil.Discard, // TODO(EVG-233) change to real logging
		Stderr:         &scpAgentCmdStderr,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}

	// get the agent's revision before scp'ing over the executable
	preSCPAgentRevision, err := self.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting pre scp agent "+
			"revision: %v", err)
	}

	// run the command to scp the agent with a timeout
	err = util.RunFunctionWithTimeout(
		scpAgentCmd.Run,
		SCPTimeout,
	)
	if err != nil {
		if err == util.ErrTimedOut {
			scpAgentCmd.Stop()
			return "", fmt.Errorf("scp-ing agent binary timed out")
		}
		return "", fmt.Errorf("error (%v) copying agent binary to remote "+
			"machine: %v", err, scpAgentCmdStderr.String())
	}

	// get the agent's revision after scp'ing over the executable
	postSCPAgentRevision, err := self.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting post scp agent "+
			"revision: %v", err)
	}

	if preSCPAgentRevision != postSCPAgentRevision {
		evergreen.Logger.Logf(slogger.WARN, "Agent revision was %v before scp "+
			"but is now %v. Using previous revision %v for host %v",
			preSCPAgentRevision, postSCPAgentRevision, preSCPAgentRevision,
			hostObj.Id)
	}

	return preSCPAgentRevision, nil
}
Beispiel #12
0
// Prepare the remote machine to run a task.
// This consists of:
// 1. Creating the directories on the remote host where, according to the distro's settings,
//    the agent should be placed.
// 2. Copying the agent into that directory.
func (agbh *AgentHostGateway) prepRemoteHost(hostObj host.Host, sshOptions []string) (string, error) {
	// compute any info necessary to ssh into the host
	hostInfo, err := util.ParseSSHInfo(hostObj.Host)
	if err != nil {
		return "", fmt.Errorf("error parsing ssh info %v: %v", hostObj.Host, err)
	}

	// first, create the necessary sandbox of directories on the remote machine
	mkdirOutput := newCappedOutputLog()
	makeShellCmd := &command.RemoteCommand{
		Id:             fmt.Sprintf("agent_mkdir-%v", rand.Int()),
		CmdString:      fmt.Sprintf("mkdir -m 777 -p %v", hostObj.Distro.WorkDir),
		Stdout:         mkdirOutput,
		Stderr:         mkdirOutput,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-p", hostInfo.Port}, sshOptions...),
		Background:     false,
	}

	evergreen.Logger.Logf(slogger.INFO, "Directories command: '%#v'", makeShellCmd)

	// run the make shell command with a timeout
	err = util.RunFunctionWithTimeout(makeShellCmd.Run, MakeShellTimeout)
	if err != nil {
		// if it timed out, kill the command
		if err == util.ErrTimedOut {
			makeShellCmd.Stop()
			return "", fmt.Errorf("creating remote directories timed out: %v", mkdirOutput.String())
		}
		return "", fmt.Errorf(
			"error creating directories on remote machine (%v): %v", err, mkdirOutput.String())
	}

	// third, copy over the correct agent binary to the remote machine
	execSubPath, err := executableSubPath(hostObj.Distro.Id)
	if err != nil {
		return "", fmt.Errorf("error computing subpath to executable: %v", err)
	}

	scpAgentOutput := newCappedOutputLog()
	scpAgentCmd := &command.ScpCommand{
		Id:             fmt.Sprintf("scp%v", rand.Int()),
		Source:         filepath.Join(agbh.ExecutablesDir, execSubPath),
		Dest:           hostObj.Distro.WorkDir,
		Stdout:         scpAgentOutput,
		Stderr:         scpAgentOutput,
		RemoteHostName: hostInfo.Hostname,
		User:           hostObj.User,
		Options:        append([]string{"-P", hostInfo.Port}, sshOptions...),
	}

	// get the agent's revision before scp'ing over the executable
	preSCPAgentRevision, err := agbh.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting pre scp agent "+
			"revision: %v", err)
	}

	// run the command to scp the agent with a timeout
	err = util.RunFunctionWithTimeout(scpAgentCmd.Run, SCPTimeout)
	if err != nil {
		if err == util.ErrTimedOut {
			scpAgentCmd.Stop()
			return "", fmt.Errorf("scp-ing agent binary timed out: %v", scpAgentOutput.String())
		}
		return "", fmt.Errorf(
			"error copying agent binary to remote machine (%v): %v", err, scpAgentOutput.String())
	}

	// get the agent's revision after scp'ing over the executable
	postSCPAgentRevision, err := agbh.GetAgentRevision()
	if err != nil {
		evergreen.Logger.Errorf(slogger.ERROR, "Error getting post scp agent "+
			"revision: %v", err)
	}

	if preSCPAgentRevision != postSCPAgentRevision {
		evergreen.Logger.Logf(slogger.WARN, "Agent revision was %v before scp "+
			"but is now %v. Using previous revision %v for host %v",
			preSCPAgentRevision, postSCPAgentRevision, preSCPAgentRevision,
			hostObj.Id)
	}

	return preSCPAgentRevision, nil
}