// Converts the plugin configuration values to environment variables
func (p *Plugin) ConfigurationToEnvironment() (*shell.Environment, error) {
env := []string{}
toDashRegex := regexp.MustCompile(`-|\s+`)
removeWhitespaceRegex := regexp.MustCompile(`\s+`)
removeDoubleUnderscore := regexp.MustCompile(`_+`)
for k, v := range p.Configuration {
k = removeWhitespaceRegex.ReplaceAllString(k, " ")
name := strings.ToUpper(toDashRegex.ReplaceAllString(fmt.Sprintf("BUILDKITE_PLUGIN_%s_%s", p.Name(), k), "_"))
name = removeDoubleUnderscore.ReplaceAllString(name, "_")
switch vv := v.(type) {
case string:
env = append(env, fmt.Sprintf("%s=%s", name, vv))
case int:
env = append(env, fmt.Sprintf("%s=%d", name, vv))
default:
// unknown type
}
}
// Sort them into a consistent order
sort.Strings(env)
return shell.EnvironmentFromSlice(env)
}
func (b *Bootstrap) Start() error {
var err error
// Create an empty env for us to keep track of our env changes in
b.env, _ = shell.EnvironmentFromSlice(os.Environ())
// Add the $BUILDKITE_BIN_PATH to the $PATH if we've been given one
if b.BinPath != "" {
b.env.Set("PATH", fmt.Sprintf("%s%s%s", b.BinPath, string(os.PathListSeparator), b.env.Get("PATH")))
}
// Come up with the place that the repository will be checked out to
var agentNameCleanupRegex = regexp.MustCompile("\"")
cleanedUpAgentName := agentNameCleanupRegex.ReplaceAllString(b.AgentName, "-")
b.env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", filepath.Join(b.BuildPath, cleanedUpAgentName, b.OrganizationSlug, b.PipelineSlug))
if b.Debug {
// Convert the env to a sorted slice
envSlice := b.env.ToSlice()
sort.Strings(envSlice)
headerf("Build environment variables")
for _, e := range envSlice {
if strings.HasPrefix(e, "BUILDKITE") || strings.HasPrefix(e, "CI") || strings.HasPrefix(e, "PATH") {
printf("%s", strings.Replace(e, "\n", "\\n", -1))
}
}
}
// Disable any interactive Git/SSH prompting
b.env.Set("GIT_TERMINAL_PROMPT", "0")
//////////////////////////////////////////////////////////////
//
// PLUGIN SETUP
//
//////////////////////////////////////////////////////////////
var plugins []*Plugin
if b.Plugins != "" {
headerf("Setting up plugins")
// Make sure we have a plugin path before trying to do anything
if b.PluginsPath == "" {
exitf("Can't checkout plugins with a `plugins-path`")
}
plugins, err = CreatePluginsFromJSON(b.Plugins)
if err != nil {
exitf("Failed to parse plugin definition (%s)", err)
}
for _, p := range plugins {
// Get the identifer for the plugin
id, err := p.Identifier()
if err != nil {
exitf("%s", err)
}
// Create a path to the plugin
directory := filepath.Join(b.PluginsPath, id)
pluginGitDirectory := filepath.Join(directory, ".git")
// Has it already been checked out?
if !fileExists(pluginGitDirectory) {
// Make the directory
err = os.MkdirAll(directory, 0777)
if err != nil {
exitf("%s", err)
}
// Try and lock this paticular plugin while we
// check it out (we create the file outside of
// the plugin directory so git clone doesn't
// have a cry about the folder not being empty)
pluginCheckoutHook, err := acquireLock(filepath.Join(b.PluginsPath, id+".lock"), 300) // Wait 5 minutes
if err != nil {
exitf("%s", err)
}
// Once we've got the lock, we need to make sure another process didn't already
// checkout the plugin
if fileExists(pluginGitDirectory) {
pluginCheckoutHook.Unlock()
commentf("Plugin \"%s\" found", p.Label())
continue
}
repo, err := p.Repository()
if err != nil {
exitf("%s", err)
}
commentf("Plugin \"%s\" will be checked out to \"%s\"", p.Location, directory)
if b.Debug {
commentf("Checking if \"%s\" is a local repository", repo)
}
// Switch to the plugin directory
previousWd := b.wd
b.wd = directory
commentf("Switching to the plugin directory")
// If it's not a local repo, and we can perform
// SSH fingerprint verification, do so.
if !fileExists(repo) && b.SSHFingerprintVerification {
b.addRepositoryHostToSSHKnownHosts(repo)
}
b.runCommand("git", "clone", "-qv", "--", repo, ".")
// Switch to the version if we need to
if p.Version != "" {
commentf("Checking out \"%s\"", p.Version)
b.runCommand("git", "checkout", "-qf", p.Version)
}
// Switch back to the previous working directory
b.wd = previousWd
// Now that we've succefully checked out the
// plugin, we can remove the lock we have on
// it.
pluginCheckoutHook.Unlock()
} else {
commentf("Plugin \"%s\" found", p.Label())
}
}
}
//////////////////////////////////////////////////////////////
//
// ENVIRONMENT SETUP
// A place for people to set up environment variables that
// might be needed for their build scripts, such as secret
// tokens and other information.
//
//////////////////////////////////////////////////////////////
// The global environment hook
b.executeGlobalHook("environment")
// The plugin environment hook
b.executePluginHook(plugins, "environment")
//////////////////////////////////////////////////////////////
//
// REPOSITORY HANDLING
// Creates the build folder and makes sure we're running the
// build at the right commit.
//
//////////////////////////////////////////////////////////////
// Run the `pre-checkout` global hook
b.executeGlobalHook("pre-checkout")
// Run the `pre-checkout` plugin hook
b.executePluginHook(plugins, "pre-checkout")
// Remove the checkout folder if BUILDKITE_CLEAN_CHECKOUT is present
if b.CleanCheckout {
headerf("Cleaning pipeline checkout")
commentf("Removing %s", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"))
err := os.RemoveAll(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"))
if err != nil {
exitf("Failed to remove \"%s\" (%s)", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"), err)
}
}
headerf("Preparing build folder")
// Create the build directory
if !fileExists(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")) {
commentf("Creating \"%s\"", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"))
os.MkdirAll(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"), 0777)
}
// Switch the internal wd to it
commentf("Switching working directroy to build folder")
b.wd = b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
// Run a custom `checkout` hook if it's present
if fileExists(b.globalHookPath("checkout")) {
b.executeGlobalHook("checkout")
} else if b.pluginHookExists(plugins, "checkout") {
b.executePluginHook(plugins, "checkout")
} else {
if b.SSHFingerprintVerification {
b.addRepositoryHostToSSHKnownHosts(b.Repository)
}
// Do we need to do a git checkout?
existingGitDir := filepath.Join(b.wd, ".git")
if fileExists(existingGitDir) {
// Update the the origin of the repository so we can
// gracefully handle repository renames
b.runCommand("git", "remote", "set-url", "origin", b.Repository)
} else {
b.runCommand("git", "clone", "-qv", "--", b.Repository, ".")
}
// Clean up the repository
b.runCommand("git", "clean", "-fdq")
// Also clean up submodules if we can
if b.GitSubmodules {
b.runCommand("git", "submodule", "foreach", "--recursive", "git", "clean", "-fdq")
}
// Allow checkouts of forked pull requests on GitHub only. See:
// https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally
if b.PullRequest != "false" && strings.Contains(b.PipelineProvider, "github") {
b.runCommand("git", "fetch", "-q", "origin", "+refs/pull/"+b.PullRequest+"/head:")
} else {
// If the commit is HEAD, we can't do a commit-only
// fetch, we'll need to use the branch instead. During
// the fetch, we do first try and grab the commit only
// (because it's usually much faster). If that doesn't
// work, just resort back to a regular fetch.
var commitToFetch string
if b.Commit == "HEAD" {
commitToFetch = b.Branch
} else {
commitToFetch = b.Commit
}
gitFetchExitStatus := b.runCommandGracefully("git", "fetch", "-q", "origin", commitToFetch)
if gitFetchExitStatus != 0 {
b.runCommand("git", "fetch", "-q")
}
// Handle checking out of tags
if b.Tag == "" {
b.runCommand("git", "reset", "--hard", "origin/"+b.Branch)
}
}
b.runCommand("git", "checkout", "-qf", b.Commit)
if b.GitSubmodules {
// `submodule sync` will ensure the .git/config
// matches the .gitmodules file. The command
// is only available in git version 1.8.1, so
// if the call fails, continue the bootstrap
// script, and show an informative error.
gitSubmoduleSyncExitStatus := b.runCommandGracefully("git", "submodule", "sync", "--recursive")
if gitSubmoduleSyncExitStatus != 0 {
gitVersionOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "--version")
warningf("Failed to recursively sync git submodules. This is most likely because you have an older version of git installed (" + gitVersionOutput + ") and you need version 1.8.1 and above. If you're using submodules, it's highly recommended you upgrade if you can.")
}
b.runCommand("git", "submodule", "update", "--init", "--recursive")
b.runCommand("git", "submodule", "foreach", "--recursive", "git", "reset", "--hard")
}
if b.env.Get("BUILDKITE_AGENT_ACCESS_TOKEN") == "" {
warningf("Skipping sending Git information to Buildkite as $BUILDKITE_AGENT_ACCESS_TOKEN is missing")
} else {
// Grab author and commit information and send
// it back to Buildkite. But before we do,
// we'll check to see if someone else has done
// it first.
commentf("Checking to see if Git data needs to be sent to Buildkite")
metaDataExistsExitStatus := b.runCommandGracefully("buildkite-agent", "meta-data", "exists", "buildkite:git:commit")
if metaDataExistsExitStatus != 0 {
commentf("Sending Git commit information back to Buildkite")
gitCommitOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "show", b.Commit, "-s", "--format=fuller", "--no-color")
gitBranchOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "branch", "--contains", b.Commit, "--no-color")
b.runCommand("buildkite-agent", "meta-data", "set", "buildkite:git:commit", gitCommitOutput)
b.runCommand("buildkite-agent", "meta-data", "set", "buildkite:git:branch", gitBranchOutput)
}
}
}
// Store the current value of BUILDKITE_BUILD_CHECKOUT_PATH, so we can detect if
// one of the post-checkout hooks changed it.
previousCheckoutPath := b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
// Run the `post-checkout` global hook
b.executeGlobalHook("post-checkout")
// Run the `post-checkout` local hook
b.executeLocalHook("post-checkout")
// Run the `post-checkout` plugin hook
b.executePluginHook(plugins, "post-checkout")
// Capture the new checkout path so we can see if it's changed. We need
// to also handle the case where they just switch it to "foo/bar",
// because that directroy is relative to the current working directroy.
newCheckoutPath := b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
newCheckoutPathAbs := newCheckoutPath
if !filepath.IsAbs(newCheckoutPathAbs) {
newCheckoutPathAbs = filepath.Join(b.wd, newCheckoutPath)
}
// If the working directory has been changed by a hook, log and switch to it
if b.wd != "" && previousCheckoutPath != newCheckoutPathAbs {
headerf("A post-checkout hook has changed the working directory to \"%s\"", newCheckoutPath)
if fileExists(newCheckoutPathAbs) {
commentf("Switching working directroy to \"%s\"", newCheckoutPathAbs)
b.wd = newCheckoutPathAbs
} else {
exitf("Failed to switch to \"%s\" as it doesn't exist", newCheckoutPathAbs)
}
}
//////////////////////////////////////////////////////////////
//
// RUN THE BUILD
// Determines how to run the build, and then runs it
//
//////////////////////////////////////////////////////////////
// Run the `pre-command` global hook
b.executeGlobalHook("pre-command")
// Run the `pre-command` local hook
b.executeLocalHook("pre-command")
// Run the `pre-command` plugin hook
b.executePluginHook(plugins, "pre-command")
var commandExitStatus int
// Run either a custom `command` hook, or the default command runner.
// We need to manually run these hooks so we can customize their
// `exitOnError` behaviour
localCommandHookPath := b.localHookPath("command")
globalCommandHookPath := b.localHookPath("command")
if fileExists(localCommandHookPath) {
commandExitStatus = b.executeHook("local command", localCommandHookPath, false, nil)
} else if fileExists(globalCommandHookPath) {
commandExitStatus = b.executeHook("global command", globalCommandHookPath, false, nil)
} else if b.pluginHookExists(plugins, "command") {
commandExitStatus = b.executePluginHookGracefully(plugins, "command")
} else {
// Make sure we actually have a command to run
if b.Command == "" {
exitf("No command has been defined. Please go to \"Pipeline Settings\" and configure your build step's \"Command\"")
}
pathToCommand := filepath.Join(b.wd, strings.Replace(b.Command, "\n", "", -1))
commandIsScript := fileExists(pathToCommand)
// If the command isn't a script, then it's something we need
// to eval. But before we even try running it, we should double
// check that the agent is allowed to eval commands.
if !commandIsScript && !b.CommandEval {
exitf("This agent is not allowed to evaluate console commands. To allow this, re-run this agent without the `--no-command-eval` option, or specify a script within your repository to run instead (such as scripts/test.sh).")
}
var headerLabel string
var buildScriptPath string
var promptDisplay string
// Come up with the contents of the build script. While we
// generate the script, we need to handle the case of running a
// script vs. a command differently
if commandIsScript {
headerLabel = "Running build script"
if runtime.GOOS == "windows" {
promptDisplay = b.Command
} else {
// Show a prettier (more accurate version) of
// what we're doing on Linux
promptDisplay = "./\"" + b.Command + "\""
}
buildScriptPath = pathToCommand
} else {
headerLabel = "Running command"
// Create a build script that will output each line of the command, and run it.
var buildScriptContents string
if runtime.GOOS == "windows" {
buildScriptContents = "@echo off\n"
for _, k := range strings.Split(b.Command, "\n") {
if k != "" {
buildScriptContents = buildScriptContents +
fmt.Sprintf("ECHO %s\n", windows.BatchEscape("\033[90m>\033[0m "+k)) +
k + "\n"
}
}
} else {
buildScriptContents = "#!/bin/bash\n"
for _, k := range strings.Split(b.Command, "\n") {
if k != "" {
buildScriptContents = buildScriptContents +
fmt.Sprintf("echo '\033[90m$\033[0m %s'\n", strings.Replace(k, "'", "'\\''", -1)) +
k + "\n"
}
}
}
// Create a temporary file where we'll run a program from
buildScriptPath = filepath.Join(b.wd, normalizeScriptFileName("buildkite-script-"+b.JobID))
if b.Debug {
headerf("Preparing build script")
commentf("A build script is being written to \"%s\" with the following:", buildScriptPath)
printf("%s", buildScriptContents)
}
// Write the build script to disk
err := ioutil.WriteFile(buildScriptPath, []byte(buildScriptContents), 0644)
if err != nil {
exitf("Failed to write to \"%s\" (%s)", buildScriptPath, err)
}
}
// Ensure it can be executed
addExecutePermissiontoFile(buildScriptPath)
// Show we're running the script
headerf("%s", headerLabel)
if promptDisplay != "" {
promptf("%s", promptDisplay)
}
commandExitStatus = b.runScript(buildScriptPath)
}
// Expand the command header if it fails
if commandExitStatus != 0 {
printf("^^^ +++")
}
// Save the command exit status to the env so hooks + plugins can access it
b.env.Set("BUILDKITE_COMMAND_EXIT_STATUS", fmt.Sprintf("%d", commandExitStatus))
// Run the `post-command` global hook
b.executeGlobalHook("post-command")
// Run the `post-command` local hook
b.executeLocalHook("post-command")
// Run the `post-command` plugin hook
b.executePluginHook(plugins, "post-command")
//////////////////////////////////////////////////////////////
//
// ARTIFACTS
// Uploads and build artifacts associated with this build
//
//////////////////////////////////////////////////////////////
if b.AutomaticArtifactUploadPaths != "" {
// Run the `pre-artifact` global hook
b.executeGlobalHook("pre-artifact")
// Run the `pre-artifact` local hook
b.executeLocalHook("pre-artifact")
// Run the `pre-artifact` plugin hook
b.executePluginHook(plugins, "pre-artifact")
// Run the artifact upload command
headerf("Uploading artifacts")
artifactUploadExitStatus := b.runCommandGracefully("buildkite-agent", "artifact", "upload", b.AutomaticArtifactUploadPaths, b.ArtifactUploadDestination)
// If the artifact upload fails, open the current group and
// exit with an error
if artifactUploadExitStatus != 0 {
printf("^^^ +++")
os.Exit(1)
}
// Run the `post-artifact` global hook
b.executeGlobalHook("post-artifact")
// Run the `post-artifact` local hook
b.executeLocalHook("post-artifact")
// Run the `post-artifact` plugin hook
b.executePluginHook(plugins, "post-artifact")
}
// Be sure to exit this script with the same exit status that the users
// build script exited with.
os.Exit(commandExitStatus)
return nil
}