func Run(conf *Config) error {
log.Debug("Running git hook")
builderKeyBytes, err := ioutil.ReadFile(builderKeyLocation)
if err != nil {
return fmt.Errorf("couldn't get builder key from %s (%s)", builderKeyLocation, err)
}
builderKey := string(builderKeyBytes)
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
line := scanner.Text()
oldRev, newRev, refName, err := readLine(line)
if err != nil {
return fmt.Errorf("reading STDIN (%s)", err)
}
log.Debug("read [%s,%s,%s]", oldRev, newRev, refName)
if err := receive(conf, builderKey, newRev); err != nil {
return err
}
// if we're processing a receive-pack on an existing repo, run a build
if strings.HasPrefix(conf.SSHOriginalCommand, "git-receive-pack") {
if err := build(conf, builderKey, newRev); err != nil {
return err
}
}
}
if err := scanner.Err(); err != nil {
return err
}
return nil
}
// run prints the command it will execute to the debug log, then runs it and returns the result of run
func run(cmd *exec.Cmd) error {
cmdStr := strings.Join(cmd.Args, " ")
if cmd.Dir != "" {
log.Debug("running [%s] in directory %s", cmdStr, cmd.Dir)
} else {
log.Debug("running [%s]", cmdStr)
}
return cmd.Run()
}
func getAppConfig(conf *Config, builderKey, userName, appName string) (*pkg.Config, error) {
url := controllerURLStr(conf, "v2", "hooks", "config")
data, err := json.Marshal(&pkg.ConfigHook{
ReceiveUser: userName,
ReceiveRepo: appName,
})
if err != nil {
return nil, err
}
b := bytes.NewReader(data)
req, err := http.NewRequest("POST", url, b)
if err != nil {
return nil, err
}
setReqHeaders(builderKey, req)
log.Debug("Workflow request POST /v2/hooks/config\n%s", string(data))
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, newUnexpectedControllerStatusCode(url, 200, res.StatusCode)
}
ret := &pkg.Config{}
if err := json.NewDecoder(res.Body).Decode(ret); err != nil {
return nil, err
}
return ret, nil
}
func receive(conf *Config, builderKey, gitSha string) error {
urlStr := controllerURLStr(conf, "v2", "hooks", "push")
bodyMap := map[string]string{
"receive_user": conf.Username,
"receive_repo": conf.App(),
"sha": gitSha,
"fingerprint": conf.Fingerprint,
"ssh_connection": conf.SSHConnection,
"ssh_original_command": conf.SSHOriginalCommand,
}
var body bytes.Buffer
if err := json.NewEncoder(&body).Encode(bodyMap); err != nil {
return err
}
log.Debug("Workflow request /v2/hooks/push (body elided)")
req, err := http.NewRequest("POST", urlStr, &body)
if err != nil {
return err
}
setReqHeaders(builderKey, req)
// TODO: use ctxhttp here (https://godoc.org/golang.org/x/net/context/ctxhttp)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 201 {
return newUnexpectedControllerStatusCode(urlStr, 201, resp.StatusCode)
}
return nil
}
func main() {
if os.Getenv("DEBUG") == "true" {
pkglog.IsDebugging = true
cookoolog.Level = cookoolog.LogDebug
}
pkglog.Debug("Running in debug mode")
app := cli.NewApp()
app.Commands = []cli.Command{
{
Name: "server",
Aliases: []string{"srv"},
Usage: "Run the git server",
Action: func(c *cli.Context) {
cnf := new(sshd.Config)
if err := conf.EnvConfig(serverConfAppName, cnf); err != nil {
pkglog.Err("getting config for %s [%s]", serverConfAppName, err)
os.Exit(1)
}
pkglog.Info("starting fetcher on port %d", cnf.FetcherPort)
go fetcher.Serve(cnf.FetcherPort)
pkglog.Info("starting SSH server on %s:%d", cnf.SSHHostIP, cnf.SSHHostPort)
os.Exit(pkg.Run(cnf.SSHHostIP, cnf.SSHHostPort, "boot"))
},
},
{
Name: "git-receive",
Aliases: []string{"gr"},
Usage: "Run the git-receive hook",
Action: func(c *cli.Context) {
cnf := new(gitreceive.Config)
if err := conf.EnvConfig(gitReceiveConfAppName, cnf); err != nil {
pkglog.Err("Error getting config for %s [%s]", gitReceiveConfAppName, err)
os.Exit(1)
}
if err := gitreceive.Run(cnf); err != nil {
pkglog.Err("running git receive hook [%s]", err)
os.Exit(1)
}
},
},
}
app.Run(os.Args)
}
func publishRelease(conf *Config, builderKey string, buildHook *pkg.BuildHook) (*pkg.BuildHookResponse, error) {
var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(buildHook); err != nil {
return nil, err
}
postBody := strings.Replace(string(b.Bytes()), "'", "", -1)
if potentialExploit.MatchString(postBody) {
return nil, fmt.Errorf("an environment variable in the app is trying to exploit Shellshock")
}
url := controllerURLStr(conf, "v2", "hooks", "build")
log.Debug("Workflow request POST /v2/hooks/build\n%s", postBody)
req, err := http.NewRequest("POST", url, strings.NewReader(postBody))
if err != nil {
return nil, err
}
setReqHeaders(builderKey, req)
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, newUnexpectedControllerStatusCode(url, 200, res.StatusCode)
}
ret := new(pkg.BuildHookResponse)
if err := json.NewDecoder(res.Body).Decode(ret); err != nil {
return nil, err
}
return ret, nil
}
func build(conf *Config, builderKey, gitSha string) error {
storage, err := getStorageConfig()
if err != nil {
return err
}
creds, err := getStorageCreds()
if err == errMissingKey || err == errMissingSecret {
return err
}
repo := conf.Repository
if len(gitSha) <= shortShaIdx {
return errGitShaTooShort{sha: gitSha}
}
shortSha := gitSha[0:8]
appName := conf.App()
repoDir := filepath.Join(conf.GitHome, repo)
buildDir := filepath.Join(repoDir, "build")
slugName := fmt.Sprintf("%s:git-%s", appName, shortSha)
imageName := strings.Replace(slugName, ":", "-", -1)
if err := os.MkdirAll(buildDir, os.ModeDir); err != nil {
return fmt.Errorf("making the build directory %s (%s)", buildDir, err)
}
tmpDir := os.TempDir()
tarURL := fmt.Sprintf("%s://%s:%s/git/home/%s/tar", storage.schema(), storage.host(), storage.port(), slugName)
// this is where workflow tells slugrunner to download the slug from, so we have to tell slugbuilder to upload it to here
pushURL := fmt.Sprintf("%s://%s:%s/git/home/%s/push", storage.schema(), storage.host(), storage.port(), fmt.Sprintf("%s:git-%s", appName, gitSha))
// Get the application config from the controller, so we can check for a custom buildpack URL
appConf, err := getAppConfig(conf, builderKey, conf.Username, appName)
if err != nil {
return fmt.Errorf("getting app config for %s (%s)", appName, err)
}
log.Debug("got the following config back for app %s: %+v", appName, *appConf)
var buildPackURL string
if buildPackURLInterface, ok := appConf.Values["BUILDPACK_URL"]; ok {
if bpStr, ok := buildPackURLInterface.(string); ok {
log.Debug("found custom buildpack URL %s", bpStr)
buildPackURL = bpStr
}
}
// build a tarball from the new objects
appTgz := fmt.Sprintf("%s.tar.gz", appName)
gitArchiveCmd := repoCmd(repoDir, "git", "archive", "--format=tar.gz", fmt.Sprintf("--output=%s", appTgz), gitSha)
gitArchiveCmd.Stdout = os.Stdout
gitArchiveCmd.Stderr = os.Stderr
if err := run(gitArchiveCmd); err != nil {
return fmt.Errorf("running %s (%s)", strings.Join(gitArchiveCmd.Args, " "), err)
}
// untar the archive into the temp dir
tarCmd := repoCmd(repoDir, "tar", "-xzf", appTgz, "-C", fmt.Sprintf("%s/", tmpDir))
tarCmd.Stdout = os.Stdout
tarCmd.Stderr = os.Stderr
if err := run(tarCmd); err != nil {
return fmt.Errorf("running %s (%s)", strings.Join(tarCmd.Args, " "), err)
}
usingDockerfile := true
rawProcFile, err := ioutil.ReadFile(fmt.Sprintf("%s/Procfile", tmpDir))
if err == nil {
usingDockerfile = false
}
var procType pkg.ProcessType
if err := yaml.Unmarshal(rawProcFile, &procType); err != nil {
return fmt.Errorf("procfile %s/ProcFile is malformed (%s)", tmpDir, err)
}
var srcManifest string
if err == os.ErrNotExist {
// both key and secret are missing, proceed with no credentials
if usingDockerfile {
srcManifest = "/etc/deis-dockerbuilder-no-creds.yaml"
} else {
srcManifest = "/etc/deis-slugbuilder-no-creds.yaml"
}
} else if err == nil {
// both key and secret are in place, so proceed with credentials
if usingDockerfile {
srcManifest = "/etc/deis-dockerbuilder.yaml"
} else {
srcManifest = "/etc/deis-slugbuilder.yaml"
}
} else if err != nil {
// unexpected error, fail
return fmt.Errorf("unexpected error (%s)", err)
}
fileBytes, err := ioutil.ReadFile(srcManifest)
if err != nil {
return fmt.Errorf("reading kubernetes manifest %s (%s)", srcManifest, err)
}
finalManifestFileLocation := fmt.Sprintf("/etc/%s", slugName)
var buildPodName string
var finalManifest string
uid := uuid.New()[:8]
if usingDockerfile {
buildPodName = fmt.Sprintf("dockerbuild-%s-%s-%s", appName, shortSha, uid)
finalManifest = strings.Replace(string(fileBytes), "repo_name", buildPodName, -1)
finalManifest = strings.Replace(finalManifest, "puturl", pushURL, -1)
finalManifest = strings.Replace(finalManifest, "tar-url", tarURL, -1)
} else {
buildPodName = fmt.Sprintf("slugbuild-%s-%s-%s", appName, shortSha, uid)
finalManifest = strings.Replace(string(fileBytes), "repo_name", buildPodName, -1)
finalManifest = strings.Replace(finalManifest, "puturl", pushURL, -1)
finalManifest = strings.Replace(finalManifest, "tar-url", tarURL, -1)
finalManifest = strings.Replace(finalManifest, "buildurl", buildPackURL, -1)
}
log.Debug("writing builder manifest to %s", finalManifestFileLocation)
if err := ioutil.WriteFile(finalManifestFileLocation, []byte(finalManifest), os.ModePerm); err != nil {
return fmt.Errorf("writing final manifest %s (%s)", finalManifestFileLocation, err)
}
configDir := "/var/minio-conf"
if err := os.MkdirAll(configDir, os.ModePerm); err != nil {
return fmt.Errorf("creating minio config file (%s)", err)
}
configCmd := mcCmd(configDir, "config", "host", "add", fmt.Sprintf("%s://%s:%s", storage.schema(), storage.host(), storage.port()), creds.key, creds.secret)
if err := run(configCmd); err != nil {
return fmt.Errorf("configuring the minio client (%s)", err)
}
makeBucketCmd := mcCmd(configDir, "mb", fmt.Sprintf("%s://%s:%s/git", storage.schema(), storage.host(), storage.port()))
// Don't look for errors here. Buckets may already exist
// https://github.com/deis/builder/issues/80 will eliminate this distaste
run(makeBucketCmd)
cpCmd := mcCmd(configDir, "cp", appTgz, tarURL)
cpCmd.Dir = repoDir
if err := run(cpCmd); err != nil {
return fmt.Errorf("copying %s to %s (%s)", appTgz, tarURL, err)
}
log.Info("Starting build... but first, coffee!")
log.Debug("Starting pod %s", buildPodName)
kCreateCmd := exec.Command(
"kubectl",
fmt.Sprintf("--namespace=%s", conf.PodNamespace),
"create",
"-f",
finalManifestFileLocation,
)
if log.IsDebugging {
kCreateCmd.Stdout = os.Stdout
}
kCreateCmd.Stderr = os.Stderr
if err := run(kCreateCmd); err != nil {
return fmt.Errorf("creating builder pod (%s)", err)
}
// poll kubectl every 100ms to determine when the build pod is running
// TODO: use the k8s client and watch the event stream instead (https://github.com/deis/builder/issues/65)
for {
cmd := kGetCmd(conf.PodNamespace, buildPodName)
var out bytes.Buffer
cmd.Stdout = &out
// ignore errors
run(cmd)
outStr := string(out.Bytes())
if strings.Contains(outStr, "phase: Running") {
break
} else if strings.Contains(outStr, "phase: Failed") {
return fmt.Errorf("build pod %s entered phase: Failed", buildPodName)
}
time.Sleep(100 * time.Millisecond)
}
// get logs from the builder pod
kLogsCmd := exec.Command(
"kubectl",
fmt.Sprintf("--namespace=%s", conf.PodNamespace),
"logs",
"-f",
buildPodName,
)
kLogsCmd.Stdout = os.Stdout
if err := run(kLogsCmd); err != nil {
return fmt.Errorf("running %s to get builder logs (%s)", strings.Join(kLogsCmd.Args, " "), err)
}
// poll the s3 server to ensure the slug exists
for {
// for now, assume the error indicates that the slug wasn't there, nothing else
// TODO: implement https://github.com/deis/builder/issues/80, which will clean this up siginficantly
lsCmd := mcCmd(configDir, "ls", pushURL)
if err := run(lsCmd); err == nil {
break
}
}
log.Info("Build complete.")
log.Info("Launching app.")
log.Info("Launching...")
buildHook := &pkg.BuildHook{
Sha: gitSha,
ReceiveUser: conf.Username,
ReceiveRepo: appName,
Image: appName,
Procfile: procType,
}
if !usingDockerfile {
buildHook.Dockerfile = ""
} else {
buildHook.Dockerfile = "true"
buildHook.Image = imageName
}
buildHookResp, err := publishRelease(conf, builderKey, buildHook)
if err != nil {
return fmt.Errorf("publishing release (%s)", err)
}
release, ok := buildHookResp.Release["version"]
if !ok {
return fmt.Errorf("No release returned from Deis controller")
}
log.Info("Done, %s:v%d deployed to Deis\n", appName, release)
log.Info("Use 'deis open' to view this application in your browser\n")
log.Info("To learn more, use 'deis help' or visit http://deis.io\n")
gcCmd := repoCmd(repoDir, "git", "gc")
if err := run(gcCmd); err != nil {
return fmt.Errorf("cleaning up the repository with %s (%s)", strings.Join(gcCmd.Args, " "), err)
}
return nil
}