Exemplo n.º 1
0
// Converts the plugin configuration values to environment variables
func (p *Plugin) ConfigurationToEnvironment() (*shell.Environment, error) {
	env := []string{}

	toDashRegex := regexp.MustCompile(`-|\s+`)
	removeWhitespaceRegex := regexp.MustCompile(`\s+`)
	removeDoubleUnderscore := regexp.MustCompile(`_+`)

	for k, v := range p.Configuration {
		k = removeWhitespaceRegex.ReplaceAllString(k, " ")
		name := strings.ToUpper(toDashRegex.ReplaceAllString(fmt.Sprintf("BUILDKITE_PLUGIN_%s_%s", p.Name(), k), "_"))
		name = removeDoubleUnderscore.ReplaceAllString(name, "_")

		switch vv := v.(type) {
		case string:
			env = append(env, fmt.Sprintf("%s=%s", name, vv))
		case int:
			env = append(env, fmt.Sprintf("%s=%d", name, vv))
		default:
			// unknown type
		}
	}

	// Sort them into a consistent order
	sort.Strings(env)

	return shell.EnvironmentFromSlice(env)
}
Exemplo n.º 2
0
func (b *Bootstrap) Start() error {
	var err error

	// Create an empty env for us to keep track of our env changes in
	b.env, _ = shell.EnvironmentFromSlice(os.Environ())

	// Add the $BUILDKITE_BIN_PATH to the $PATH if we've been given one
	if b.BinPath != "" {
		b.env.Set("PATH", fmt.Sprintf("%s%s%s", b.BinPath, string(os.PathListSeparator), b.env.Get("PATH")))
	}

	// Come up with the place that the repository will be checked out to
	var agentNameCleanupRegex = regexp.MustCompile("\"")
	cleanedUpAgentName := agentNameCleanupRegex.ReplaceAllString(b.AgentName, "-")

	b.env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", filepath.Join(b.BuildPath, cleanedUpAgentName, b.OrganizationSlug, b.PipelineSlug))

	if b.Debug {
		// Convert the env to a sorted slice
		envSlice := b.env.ToSlice()
		sort.Strings(envSlice)

		headerf("Build environment variables")
		for _, e := range envSlice {
			if strings.HasPrefix(e, "BUILDKITE") || strings.HasPrefix(e, "CI") || strings.HasPrefix(e, "PATH") {
				printf("%s", strings.Replace(e, "\n", "\\n", -1))
			}
		}
	}

	// Disable any interactive Git/SSH prompting
	b.env.Set("GIT_TERMINAL_PROMPT", "0")

	//////////////////////////////////////////////////////////////
	//
	// PLUGIN SETUP
	//
	//////////////////////////////////////////////////////////////

	var plugins []*Plugin

	if b.Plugins != "" {
		headerf("Setting up plugins")

		// Make sure we have a plugin path before trying to do anything
		if b.PluginsPath == "" {
			exitf("Can't checkout plugins with a `plugins-path`")
		}

		plugins, err = CreatePluginsFromJSON(b.Plugins)
		if err != nil {
			exitf("Failed to parse plugin definition (%s)", err)
		}

		for _, p := range plugins {
			// Get the identifer for the plugin
			id, err := p.Identifier()
			if err != nil {
				exitf("%s", err)
			}

			// Create a path to the plugin
			directory := filepath.Join(b.PluginsPath, id)
			pluginGitDirectory := filepath.Join(directory, ".git")

			// Has it already been checked out?
			if !fileExists(pluginGitDirectory) {
				// Make the directory
				err = os.MkdirAll(directory, 0777)
				if err != nil {
					exitf("%s", err)
				}

				// Try and lock this paticular plugin while we
				// check it out (we create the file outside of
				// the plugin directory so git clone doesn't
				// have a cry about the folder not being empty)
				pluginCheckoutHook, err := acquireLock(filepath.Join(b.PluginsPath, id+".lock"), 300) // Wait 5 minutes
				if err != nil {
					exitf("%s", err)
				}

				// Once we've got the lock, we need to make sure another process didn't already
				// checkout the plugin
				if fileExists(pluginGitDirectory) {
					pluginCheckoutHook.Unlock()
					commentf("Plugin \"%s\" found", p.Label())
					continue
				}

				repo, err := p.Repository()
				if err != nil {
					exitf("%s", err)
				}

				commentf("Plugin \"%s\" will be checked out to \"%s\"", p.Location, directory)

				if b.Debug {
					commentf("Checking if \"%s\" is a local repository", repo)
				}

				// Switch to the plugin directory
				previousWd := b.wd
				b.wd = directory

				commentf("Switching to the plugin directory")

				// If it's not a local repo, and we can perform
				// SSH fingerprint verification, do so.
				if !fileExists(repo) && b.SSHFingerprintVerification {
					b.addRepositoryHostToSSHKnownHosts(repo)
				}

				b.runCommand("git", "clone", "-qv", "--", repo, ".")

				// Switch to the version if we need to
				if p.Version != "" {
					commentf("Checking out \"%s\"", p.Version)
					b.runCommand("git", "checkout", "-qf", p.Version)
				}

				// Switch back to the previous working directory
				b.wd = previousWd

				// Now that we've succefully checked out the
				// plugin, we can remove the lock we have on
				// it.
				pluginCheckoutHook.Unlock()
			} else {
				commentf("Plugin \"%s\" found", p.Label())
			}
		}
	}

	//////////////////////////////////////////////////////////////
	//
	// ENVIRONMENT SETUP
	// A place for people to set up environment variables that
	// might be needed for their build scripts, such as secret
	// tokens and other information.
	//
	//////////////////////////////////////////////////////////////

	// The global environment hook
	b.executeGlobalHook("environment")

	// The plugin environment hook
	b.executePluginHook(plugins, "environment")

	//////////////////////////////////////////////////////////////
	//
	// REPOSITORY HANDLING
	// Creates the build folder and makes sure we're running the
	// build at the right commit.
	//
	//////////////////////////////////////////////////////////////

	// Run the `pre-checkout` global hook
	b.executeGlobalHook("pre-checkout")

	// Run the `pre-checkout` plugin hook
	b.executePluginHook(plugins, "pre-checkout")

	// Remove the checkout folder if BUILDKITE_CLEAN_CHECKOUT is present
	if b.CleanCheckout {
		headerf("Cleaning pipeline checkout")
		commentf("Removing %s", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"))

		err := os.RemoveAll(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"))
		if err != nil {
			exitf("Failed to remove \"%s\" (%s)", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"), err)
		}
	}

	headerf("Preparing build folder")

	// Create the build directory
	if !fileExists(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")) {
		commentf("Creating \"%s\"", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"))
		os.MkdirAll(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"), 0777)
	}

	// Switch the internal wd to it
	commentf("Switching working directroy to build folder")
	b.wd = b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")

	// Run a custom `checkout` hook if it's present
	if fileExists(b.globalHookPath("checkout")) {
		b.executeGlobalHook("checkout")
	} else if b.pluginHookExists(plugins, "checkout") {
		b.executePluginHook(plugins, "checkout")
	} else {
		if b.SSHFingerprintVerification {
			b.addRepositoryHostToSSHKnownHosts(b.Repository)
		}

		// Do we need to do a git checkout?
		existingGitDir := filepath.Join(b.wd, ".git")
		if fileExists(existingGitDir) {
			// Update the the origin of the repository so we can
			// gracefully handle repository renames
			b.runCommand("git", "remote", "set-url", "origin", b.Repository)
		} else {
			b.runCommand("git", "clone", "-qv", "--", b.Repository, ".")
		}

		// Clean up the repository
		b.runCommand("git", "clean", "-fdq")

		// Also clean up submodules if we can
		if b.GitSubmodules {
			b.runCommand("git", "submodule", "foreach", "--recursive", "git", "clean", "-fdq")
		}

		// Allow checkouts of forked pull requests on GitHub only. See:
		// https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally
		if b.PullRequest != "false" && strings.Contains(b.PipelineProvider, "github") {
			b.runCommand("git", "fetch", "-q", "origin", "+refs/pull/"+b.PullRequest+"/head:")
		} else {
			// If the commit is HEAD, we can't do a commit-only
			// fetch, we'll need to use the branch instead.  During
			// the fetch, we do first try and grab the commit only
			// (because it's usually much faster).  If that doesn't
			// work, just resort back to a regular fetch.
			var commitToFetch string
			if b.Commit == "HEAD" {
				commitToFetch = b.Branch
			} else {
				commitToFetch = b.Commit
			}

			gitFetchExitStatus := b.runCommandGracefully("git", "fetch", "-q", "origin", commitToFetch)
			if gitFetchExitStatus != 0 {
				b.runCommand("git", "fetch", "-q")
			}

			// Handle checking out of tags
			if b.Tag == "" {
				b.runCommand("git", "reset", "--hard", "origin/"+b.Branch)
			}
		}

		b.runCommand("git", "checkout", "-qf", b.Commit)

		if b.GitSubmodules {
			// `submodule sync` will ensure the .git/config
			// matches the .gitmodules file.  The command
			// is only available in git version 1.8.1, so
			// if the call fails, continue the bootstrap
			// script, and show an informative error.
			gitSubmoduleSyncExitStatus := b.runCommandGracefully("git", "submodule", "sync", "--recursive")
			if gitSubmoduleSyncExitStatus != 0 {
				gitVersionOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "--version")
				warningf("Failed to recursively sync git submodules. This is most likely because you have an older version of git installed (" + gitVersionOutput + ") and you need version 1.8.1 and above. If you're using submodules, it's highly recommended you upgrade if you can.")
			}

			b.runCommand("git", "submodule", "update", "--init", "--recursive")
			b.runCommand("git", "submodule", "foreach", "--recursive", "git", "reset", "--hard")
		}

		if b.env.Get("BUILDKITE_AGENT_ACCESS_TOKEN") == "" {
			warningf("Skipping sending Git information to Buildkite as $BUILDKITE_AGENT_ACCESS_TOKEN is missing")
		} else {
			// Grab author and commit information and send
			// it back to Buildkite. But before we do,
			// we'll check to see if someone else has done
			// it first.
			commentf("Checking to see if Git data needs to be sent to Buildkite")
			metaDataExistsExitStatus := b.runCommandGracefully("buildkite-agent", "meta-data", "exists", "buildkite:git:commit")
			if metaDataExistsExitStatus != 0 {
				commentf("Sending Git commit information back to Buildkite")

				gitCommitOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "show", b.Commit, "-s", "--format=fuller", "--no-color")
				gitBranchOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "branch", "--contains", b.Commit, "--no-color")

				b.runCommand("buildkite-agent", "meta-data", "set", "buildkite:git:commit", gitCommitOutput)
				b.runCommand("buildkite-agent", "meta-data", "set", "buildkite:git:branch", gitBranchOutput)
			}
		}
	}

	// Store the current value of BUILDKITE_BUILD_CHECKOUT_PATH, so we can detect if
	// one of the post-checkout hooks changed it.
	previousCheckoutPath := b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")

	// Run the `post-checkout` global hook
	b.executeGlobalHook("post-checkout")

	// Run the `post-checkout` local hook
	b.executeLocalHook("post-checkout")

	// Run the `post-checkout` plugin hook
	b.executePluginHook(plugins, "post-checkout")

	// Capture the new checkout path so we can see if it's changed. We need
	// to also handle the case where they just switch it to "foo/bar",
	// because that directroy is relative to the current working directroy.
	newCheckoutPath := b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
	newCheckoutPathAbs := newCheckoutPath
	if !filepath.IsAbs(newCheckoutPathAbs) {
		newCheckoutPathAbs = filepath.Join(b.wd, newCheckoutPath)
	}

	// If the working directory has been changed by a hook, log and switch to it
	if b.wd != "" && previousCheckoutPath != newCheckoutPathAbs {
		headerf("A post-checkout hook has changed the working directory to \"%s\"", newCheckoutPath)

		if fileExists(newCheckoutPathAbs) {
			commentf("Switching working directroy to \"%s\"", newCheckoutPathAbs)
			b.wd = newCheckoutPathAbs
		} else {
			exitf("Failed to switch to \"%s\" as it doesn't exist", newCheckoutPathAbs)
		}
	}

	//////////////////////////////////////////////////////////////
	//
	// RUN THE BUILD
	// Determines how to run the build, and then runs it
	//
	//////////////////////////////////////////////////////////////

	// Run the `pre-command` global hook
	b.executeGlobalHook("pre-command")

	// Run the `pre-command` local hook
	b.executeLocalHook("pre-command")

	// Run the `pre-command` plugin hook
	b.executePluginHook(plugins, "pre-command")

	var commandExitStatus int

	// Run either a custom `command` hook, or the default command runner.
	// We need to manually run these hooks so we can customize their
	// `exitOnError` behaviour
	localCommandHookPath := b.localHookPath("command")
	globalCommandHookPath := b.localHookPath("command")

	if fileExists(localCommandHookPath) {
		commandExitStatus = b.executeHook("local command", localCommandHookPath, false, nil)
	} else if fileExists(globalCommandHookPath) {
		commandExitStatus = b.executeHook("global command", globalCommandHookPath, false, nil)
	} else if b.pluginHookExists(plugins, "command") {
		commandExitStatus = b.executePluginHookGracefully(plugins, "command")
	} else {
		// Make sure we actually have a command to run
		if b.Command == "" {
			exitf("No command has been defined. Please go to \"Pipeline Settings\" and configure your build step's \"Command\"")
		}

		pathToCommand := filepath.Join(b.wd, strings.Replace(b.Command, "\n", "", -1))
		commandIsScript := fileExists(pathToCommand)

		// If the command isn't a script, then it's something we need
		// to eval. But before we even try running it, we should double
		// check that the agent is allowed to eval commands.
		if !commandIsScript && !b.CommandEval {
			exitf("This agent is not allowed to evaluate console commands. To allow this, re-run this agent without the `--no-command-eval` option, or specify a script within your repository to run instead (such as scripts/test.sh).")
		}

		var headerLabel string
		var buildScriptPath string
		var promptDisplay string

		// Come up with the contents of the build script. While we
		// generate the script, we need to handle the case of running a
		// script vs. a command differently
		if commandIsScript {
			headerLabel = "Running build script"

			if runtime.GOOS == "windows" {
				promptDisplay = b.Command
			} else {
				// Show a prettier (more accurate version) of
				// what we're doing on Linux
				promptDisplay = "./\"" + b.Command + "\""
			}

			buildScriptPath = pathToCommand
		} else {
			headerLabel = "Running command"

			// Create a build script that will output each line of the command, and run it.
			var buildScriptContents string
			if runtime.GOOS == "windows" {
				buildScriptContents = "@echo off\n"
				for _, k := range strings.Split(b.Command, "\n") {
					if k != "" {
						buildScriptContents = buildScriptContents +
							fmt.Sprintf("ECHO %s\n", windows.BatchEscape("\033[90m>\033[0m "+k)) +
							k + "\n"
					}
				}
			} else {
				buildScriptContents = "#!/bin/bash\n"
				for _, k := range strings.Split(b.Command, "\n") {
					if k != "" {
						buildScriptContents = buildScriptContents +
							fmt.Sprintf("echo '\033[90m$\033[0m %s'\n", strings.Replace(k, "'", "'\\''", -1)) +
							k + "\n"
					}
				}
			}

			// Create a temporary file where we'll run a program from
			buildScriptPath = filepath.Join(b.wd, normalizeScriptFileName("buildkite-script-"+b.JobID))

			if b.Debug {
				headerf("Preparing build script")
				commentf("A build script is being written to \"%s\" with the following:", buildScriptPath)
				printf("%s", buildScriptContents)
			}

			// Write the build script to disk
			err := ioutil.WriteFile(buildScriptPath, []byte(buildScriptContents), 0644)
			if err != nil {
				exitf("Failed to write to \"%s\" (%s)", buildScriptPath, err)
			}
		}

		// Ensure it can be executed
		addExecutePermissiontoFile(buildScriptPath)

		// Show we're running the script
		headerf("%s", headerLabel)
		if promptDisplay != "" {
			promptf("%s", promptDisplay)
		}

		commandExitStatus = b.runScript(buildScriptPath)
	}

	// Expand the command header if it fails
	if commandExitStatus != 0 {
		printf("^^^ +++")
	}

	// Save the command exit status to the env so hooks + plugins can access it
	b.env.Set("BUILDKITE_COMMAND_EXIT_STATUS", fmt.Sprintf("%d", commandExitStatus))

	// Run the `post-command` global hook
	b.executeGlobalHook("post-command")

	// Run the `post-command` local hook
	b.executeLocalHook("post-command")

	// Run the `post-command` plugin hook
	b.executePluginHook(plugins, "post-command")

	//////////////////////////////////////////////////////////////
	//
	// ARTIFACTS
	// Uploads and build artifacts associated with this build
	//
	//////////////////////////////////////////////////////////////

	if b.AutomaticArtifactUploadPaths != "" {
		// Run the `pre-artifact` global hook
		b.executeGlobalHook("pre-artifact")

		// Run the `pre-artifact` local hook
		b.executeLocalHook("pre-artifact")

		// Run the `pre-artifact` plugin hook
		b.executePluginHook(plugins, "pre-artifact")

		// Run the artifact upload command
		headerf("Uploading artifacts")
		artifactUploadExitStatus := b.runCommandGracefully("buildkite-agent", "artifact", "upload", b.AutomaticArtifactUploadPaths, b.ArtifactUploadDestination)

		// If the artifact upload fails, open the current group and
		// exit with an error
		if artifactUploadExitStatus != 0 {
			printf("^^^ +++")
			os.Exit(1)
		}

		// Run the `post-artifact` global hook
		b.executeGlobalHook("post-artifact")

		// Run the `post-artifact` local hook
		b.executeLocalHook("post-artifact")

		// Run the `post-artifact` plugin hook
		b.executePluginHook(plugins, "post-artifact")
	}

	// Be sure to exit this script with the same exit status that the users
	// build script exited with.
	os.Exit(commandExitStatus)

	return nil
}