// Converts the plugin configuration values to environment variables func (p *Plugin) ConfigurationToEnvironment() (*shell.Environment, error) { env := []string{} toDashRegex := regexp.MustCompile(`-|\s+`) removeWhitespaceRegex := regexp.MustCompile(`\s+`) removeDoubleUnderscore := regexp.MustCompile(`_+`) for k, v := range p.Configuration { k = removeWhitespaceRegex.ReplaceAllString(k, " ") name := strings.ToUpper(toDashRegex.ReplaceAllString(fmt.Sprintf("BUILDKITE_PLUGIN_%s_%s", p.Name(), k), "_")) name = removeDoubleUnderscore.ReplaceAllString(name, "_") switch vv := v.(type) { case string: env = append(env, fmt.Sprintf("%s=%s", name, vv)) case int: env = append(env, fmt.Sprintf("%s=%d", name, vv)) default: // unknown type } } // Sort them into a consistent order sort.Strings(env) return shell.EnvironmentFromSlice(env) }
func (b *Bootstrap) Start() error { var err error // Create an empty env for us to keep track of our env changes in b.env, _ = shell.EnvironmentFromSlice(os.Environ()) // Add the $BUILDKITE_BIN_PATH to the $PATH if we've been given one if b.BinPath != "" { b.env.Set("PATH", fmt.Sprintf("%s%s%s", b.BinPath, string(os.PathListSeparator), b.env.Get("PATH"))) } // Come up with the place that the repository will be checked out to var agentNameCleanupRegex = regexp.MustCompile("\"") cleanedUpAgentName := agentNameCleanupRegex.ReplaceAllString(b.AgentName, "-") b.env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", filepath.Join(b.BuildPath, cleanedUpAgentName, b.OrganizationSlug, b.PipelineSlug)) if b.Debug { // Convert the env to a sorted slice envSlice := b.env.ToSlice() sort.Strings(envSlice) headerf("Build environment variables") for _, e := range envSlice { if strings.HasPrefix(e, "BUILDKITE") || strings.HasPrefix(e, "CI") || strings.HasPrefix(e, "PATH") { printf("%s", strings.Replace(e, "\n", "\\n", -1)) } } } // Disable any interactive Git/SSH prompting b.env.Set("GIT_TERMINAL_PROMPT", "0") ////////////////////////////////////////////////////////////// // // PLUGIN SETUP // ////////////////////////////////////////////////////////////// var plugins []*Plugin if b.Plugins != "" { headerf("Setting up plugins") // Make sure we have a plugin path before trying to do anything if b.PluginsPath == "" { exitf("Can't checkout plugins with a `plugins-path`") } plugins, err = CreatePluginsFromJSON(b.Plugins) if err != nil { exitf("Failed to parse plugin definition (%s)", err) } for _, p := range plugins { // Get the identifer for the plugin id, err := p.Identifier() if err != nil { exitf("%s", err) } // Create a path to the plugin directory := filepath.Join(b.PluginsPath, id) pluginGitDirectory := filepath.Join(directory, ".git") // Has it already been checked out? if !fileExists(pluginGitDirectory) { // Make the directory err = os.MkdirAll(directory, 0777) if err != nil { exitf("%s", err) } // Try and lock this paticular plugin while we // check it out (we create the file outside of // the plugin directory so git clone doesn't // have a cry about the folder not being empty) pluginCheckoutHook, err := acquireLock(filepath.Join(b.PluginsPath, id+".lock"), 300) // Wait 5 minutes if err != nil { exitf("%s", err) } // Once we've got the lock, we need to make sure another process didn't already // checkout the plugin if fileExists(pluginGitDirectory) { pluginCheckoutHook.Unlock() commentf("Plugin \"%s\" found", p.Label()) continue } repo, err := p.Repository() if err != nil { exitf("%s", err) } commentf("Plugin \"%s\" will be checked out to \"%s\"", p.Location, directory) if b.Debug { commentf("Checking if \"%s\" is a local repository", repo) } // Switch to the plugin directory previousWd := b.wd b.wd = directory commentf("Switching to the plugin directory") // If it's not a local repo, and we can perform // SSH fingerprint verification, do so. if !fileExists(repo) && b.SSHFingerprintVerification { b.addRepositoryHostToSSHKnownHosts(repo) } b.runCommand("git", "clone", "-qv", "--", repo, ".") // Switch to the version if we need to if p.Version != "" { commentf("Checking out \"%s\"", p.Version) b.runCommand("git", "checkout", "-qf", p.Version) } // Switch back to the previous working directory b.wd = previousWd // Now that we've succefully checked out the // plugin, we can remove the lock we have on // it. pluginCheckoutHook.Unlock() } else { commentf("Plugin \"%s\" found", p.Label()) } } } ////////////////////////////////////////////////////////////// // // ENVIRONMENT SETUP // A place for people to set up environment variables that // might be needed for their build scripts, such as secret // tokens and other information. // ////////////////////////////////////////////////////////////// // The global environment hook b.executeGlobalHook("environment") // The plugin environment hook b.executePluginHook(plugins, "environment") ////////////////////////////////////////////////////////////// // // REPOSITORY HANDLING // Creates the build folder and makes sure we're running the // build at the right commit. // ////////////////////////////////////////////////////////////// // Run the `pre-checkout` global hook b.executeGlobalHook("pre-checkout") // Run the `pre-checkout` plugin hook b.executePluginHook(plugins, "pre-checkout") // Remove the checkout folder if BUILDKITE_CLEAN_CHECKOUT is present if b.CleanCheckout { headerf("Cleaning pipeline checkout") commentf("Removing %s", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")) err := os.RemoveAll(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")) if err != nil { exitf("Failed to remove \"%s\" (%s)", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"), err) } } headerf("Preparing build folder") // Create the build directory if !fileExists(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")) { commentf("Creating \"%s\"", b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")) os.MkdirAll(b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH"), 0777) } // Switch the internal wd to it commentf("Switching working directroy to build folder") b.wd = b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH") // Run a custom `checkout` hook if it's present if fileExists(b.globalHookPath("checkout")) { b.executeGlobalHook("checkout") } else if b.pluginHookExists(plugins, "checkout") { b.executePluginHook(plugins, "checkout") } else { if b.SSHFingerprintVerification { b.addRepositoryHostToSSHKnownHosts(b.Repository) } // Do we need to do a git checkout? existingGitDir := filepath.Join(b.wd, ".git") if fileExists(existingGitDir) { // Update the the origin of the repository so we can // gracefully handle repository renames b.runCommand("git", "remote", "set-url", "origin", b.Repository) } else { b.runCommand("git", "clone", "-qv", "--", b.Repository, ".") } // Clean up the repository b.runCommand("git", "clean", "-fdq") // Also clean up submodules if we can if b.GitSubmodules { b.runCommand("git", "submodule", "foreach", "--recursive", "git", "clean", "-fdq") } // Allow checkouts of forked pull requests on GitHub only. See: // https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally if b.PullRequest != "false" && strings.Contains(b.PipelineProvider, "github") { b.runCommand("git", "fetch", "-q", "origin", "+refs/pull/"+b.PullRequest+"/head:") } else { // If the commit is HEAD, we can't do a commit-only // fetch, we'll need to use the branch instead. During // the fetch, we do first try and grab the commit only // (because it's usually much faster). If that doesn't // work, just resort back to a regular fetch. var commitToFetch string if b.Commit == "HEAD" { commitToFetch = b.Branch } else { commitToFetch = b.Commit } gitFetchExitStatus := b.runCommandGracefully("git", "fetch", "-q", "origin", commitToFetch) if gitFetchExitStatus != 0 { b.runCommand("git", "fetch", "-q") } // Handle checking out of tags if b.Tag == "" { b.runCommand("git", "reset", "--hard", "origin/"+b.Branch) } } b.runCommand("git", "checkout", "-qf", b.Commit) if b.GitSubmodules { // `submodule sync` will ensure the .git/config // matches the .gitmodules file. The command // is only available in git version 1.8.1, so // if the call fails, continue the bootstrap // script, and show an informative error. gitSubmoduleSyncExitStatus := b.runCommandGracefully("git", "submodule", "sync", "--recursive") if gitSubmoduleSyncExitStatus != 0 { gitVersionOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "--version") warningf("Failed to recursively sync git submodules. This is most likely because you have an older version of git installed (" + gitVersionOutput + ") and you need version 1.8.1 and above. If you're using submodules, it's highly recommended you upgrade if you can.") } b.runCommand("git", "submodule", "update", "--init", "--recursive") b.runCommand("git", "submodule", "foreach", "--recursive", "git", "reset", "--hard") } if b.env.Get("BUILDKITE_AGENT_ACCESS_TOKEN") == "" { warningf("Skipping sending Git information to Buildkite as $BUILDKITE_AGENT_ACCESS_TOKEN is missing") } else { // Grab author and commit information and send // it back to Buildkite. But before we do, // we'll check to see if someone else has done // it first. commentf("Checking to see if Git data needs to be sent to Buildkite") metaDataExistsExitStatus := b.runCommandGracefully("buildkite-agent", "meta-data", "exists", "buildkite:git:commit") if metaDataExistsExitStatus != 0 { commentf("Sending Git commit information back to Buildkite") gitCommitOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "show", b.Commit, "-s", "--format=fuller", "--no-color") gitBranchOutput, _ := b.runCommandSilentlyAndCaptureOutput("git", "branch", "--contains", b.Commit, "--no-color") b.runCommand("buildkite-agent", "meta-data", "set", "buildkite:git:commit", gitCommitOutput) b.runCommand("buildkite-agent", "meta-data", "set", "buildkite:git:branch", gitBranchOutput) } } } // Store the current value of BUILDKITE_BUILD_CHECKOUT_PATH, so we can detect if // one of the post-checkout hooks changed it. previousCheckoutPath := b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH") // Run the `post-checkout` global hook b.executeGlobalHook("post-checkout") // Run the `post-checkout` local hook b.executeLocalHook("post-checkout") // Run the `post-checkout` plugin hook b.executePluginHook(plugins, "post-checkout") // Capture the new checkout path so we can see if it's changed. We need // to also handle the case where they just switch it to "foo/bar", // because that directroy is relative to the current working directroy. newCheckoutPath := b.env.Get("BUILDKITE_BUILD_CHECKOUT_PATH") newCheckoutPathAbs := newCheckoutPath if !filepath.IsAbs(newCheckoutPathAbs) { newCheckoutPathAbs = filepath.Join(b.wd, newCheckoutPath) } // If the working directory has been changed by a hook, log and switch to it if b.wd != "" && previousCheckoutPath != newCheckoutPathAbs { headerf("A post-checkout hook has changed the working directory to \"%s\"", newCheckoutPath) if fileExists(newCheckoutPathAbs) { commentf("Switching working directroy to \"%s\"", newCheckoutPathAbs) b.wd = newCheckoutPathAbs } else { exitf("Failed to switch to \"%s\" as it doesn't exist", newCheckoutPathAbs) } } ////////////////////////////////////////////////////////////// // // RUN THE BUILD // Determines how to run the build, and then runs it // ////////////////////////////////////////////////////////////// // Run the `pre-command` global hook b.executeGlobalHook("pre-command") // Run the `pre-command` local hook b.executeLocalHook("pre-command") // Run the `pre-command` plugin hook b.executePluginHook(plugins, "pre-command") var commandExitStatus int // Run either a custom `command` hook, or the default command runner. // We need to manually run these hooks so we can customize their // `exitOnError` behaviour localCommandHookPath := b.localHookPath("command") globalCommandHookPath := b.localHookPath("command") if fileExists(localCommandHookPath) { commandExitStatus = b.executeHook("local command", localCommandHookPath, false, nil) } else if fileExists(globalCommandHookPath) { commandExitStatus = b.executeHook("global command", globalCommandHookPath, false, nil) } else if b.pluginHookExists(plugins, "command") { commandExitStatus = b.executePluginHookGracefully(plugins, "command") } else { // Make sure we actually have a command to run if b.Command == "" { exitf("No command has been defined. Please go to \"Pipeline Settings\" and configure your build step's \"Command\"") } pathToCommand := filepath.Join(b.wd, strings.Replace(b.Command, "\n", "", -1)) commandIsScript := fileExists(pathToCommand) // If the command isn't a script, then it's something we need // to eval. But before we even try running it, we should double // check that the agent is allowed to eval commands. if !commandIsScript && !b.CommandEval { exitf("This agent is not allowed to evaluate console commands. To allow this, re-run this agent without the `--no-command-eval` option, or specify a script within your repository to run instead (such as scripts/test.sh).") } var headerLabel string var buildScriptPath string var promptDisplay string // Come up with the contents of the build script. While we // generate the script, we need to handle the case of running a // script vs. a command differently if commandIsScript { headerLabel = "Running build script" if runtime.GOOS == "windows" { promptDisplay = b.Command } else { // Show a prettier (more accurate version) of // what we're doing on Linux promptDisplay = "./\"" + b.Command + "\"" } buildScriptPath = pathToCommand } else { headerLabel = "Running command" // Create a build script that will output each line of the command, and run it. var buildScriptContents string if runtime.GOOS == "windows" { buildScriptContents = "@echo off\n" for _, k := range strings.Split(b.Command, "\n") { if k != "" { buildScriptContents = buildScriptContents + fmt.Sprintf("ECHO %s\n", windows.BatchEscape("\033[90m>\033[0m "+k)) + k + "\n" } } } else { buildScriptContents = "#!/bin/bash\n" for _, k := range strings.Split(b.Command, "\n") { if k != "" { buildScriptContents = buildScriptContents + fmt.Sprintf("echo '\033[90m$\033[0m %s'\n", strings.Replace(k, "'", "'\\''", -1)) + k + "\n" } } } // Create a temporary file where we'll run a program from buildScriptPath = filepath.Join(b.wd, normalizeScriptFileName("buildkite-script-"+b.JobID)) if b.Debug { headerf("Preparing build script") commentf("A build script is being written to \"%s\" with the following:", buildScriptPath) printf("%s", buildScriptContents) } // Write the build script to disk err := ioutil.WriteFile(buildScriptPath, []byte(buildScriptContents), 0644) if err != nil { exitf("Failed to write to \"%s\" (%s)", buildScriptPath, err) } } // Ensure it can be executed addExecutePermissiontoFile(buildScriptPath) // Show we're running the script headerf("%s", headerLabel) if promptDisplay != "" { promptf("%s", promptDisplay) } commandExitStatus = b.runScript(buildScriptPath) } // Expand the command header if it fails if commandExitStatus != 0 { printf("^^^ +++") } // Save the command exit status to the env so hooks + plugins can access it b.env.Set("BUILDKITE_COMMAND_EXIT_STATUS", fmt.Sprintf("%d", commandExitStatus)) // Run the `post-command` global hook b.executeGlobalHook("post-command") // Run the `post-command` local hook b.executeLocalHook("post-command") // Run the `post-command` plugin hook b.executePluginHook(plugins, "post-command") ////////////////////////////////////////////////////////////// // // ARTIFACTS // Uploads and build artifacts associated with this build // ////////////////////////////////////////////////////////////// if b.AutomaticArtifactUploadPaths != "" { // Run the `pre-artifact` global hook b.executeGlobalHook("pre-artifact") // Run the `pre-artifact` local hook b.executeLocalHook("pre-artifact") // Run the `pre-artifact` plugin hook b.executePluginHook(plugins, "pre-artifact") // Run the artifact upload command headerf("Uploading artifacts") artifactUploadExitStatus := b.runCommandGracefully("buildkite-agent", "artifact", "upload", b.AutomaticArtifactUploadPaths, b.ArtifactUploadDestination) // If the artifact upload fails, open the current group and // exit with an error if artifactUploadExitStatus != 0 { printf("^^^ +++") os.Exit(1) } // Run the `post-artifact` global hook b.executeGlobalHook("post-artifact") // Run the `post-artifact` local hook b.executeLocalHook("post-artifact") // Run the `post-artifact` plugin hook b.executePluginHook(plugins, "post-artifact") } // Be sure to exit this script with the same exit status that the users // build script exited with. os.Exit(commandExitStatus) return nil }