// CachedRemoteRefs returns the list of branches & tags for a remote which are
// currently cached locally. No remote request is made to verify them.
func CachedRemoteRefs(remoteName string) ([]*Ref, error) {
var ret []*Ref
cmd := subprocess.ExecCommand("git", "show-ref")
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git show-ref: %v", err)
}
cmd.Start()
scanner := bufio.NewScanner(outp)
r := regexp.MustCompile(fmt.Sprintf(`([0-9a-fA-F]{40})\s+refs/remotes/%v/(.*)`, remoteName))
for scanner.Scan() {
if match := r.FindStringSubmatch(scanner.Text()); match != nil {
name := strings.TrimSpace(match[2])
// Don't match head
if name == "HEAD" {
continue
}
sha := match[1]
ret = append(ret, &Ref{name, RefTypeRemoteBranch, sha})
}
}
return ret, nil
}
// RemoteRefs returns a list of branches & tags for a remote by actually
// accessing the remote vir git ls-remote
func RemoteRefs(remoteName string) ([]*Ref, error) {
var ret []*Ref
cmd := subprocess.ExecCommand("git", "ls-remote", "--heads", "--tags", "-q", remoteName)
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git ls-remote: %v", err)
}
cmd.Start()
scanner := bufio.NewScanner(outp)
r := regexp.MustCompile(`([0-9a-fA-F]{40})\s+refs/(heads|tags)/(.*)`)
for scanner.Scan() {
if match := r.FindStringSubmatch(scanner.Text()); match != nil {
name := strings.TrimSpace(match[3])
// Don't match head
if name == "HEAD" {
continue
}
sha := match[1]
if match[2] == "heads" {
ret = append(ret, &Ref{name, RefTypeRemoteBranch, sha})
} else {
ret = append(ret, &Ref{name, RefTypeRemoteTag, sha})
}
}
}
return ret, nil
}
// Get summary information about a commit
func GetCommitSummary(commit string) (*CommitSummary, error) {
cmd := subprocess.ExecCommand("git", "show", "-s",
`--format=%H|%h|%P|%ai|%ci|%ae|%an|%ce|%cn|%s`, commit)
out, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("Failed to call git show: %v %v", err, string(out))
}
// At most 10 substrings so subject line is not split on anything
fields := strings.SplitN(string(out), "|", 10)
// Cope with the case where subject is blank
if len(fields) >= 9 {
ret := &CommitSummary{}
// Get SHAs from output, not commit input, so we can support symbolic refs
ret.Sha = fields[0]
ret.ShortSha = fields[1]
ret.Parents = strings.Split(fields[2], " ")
// %aD & %cD (RFC2822) matches Go's RFC1123Z format
ret.AuthorDate, _ = ParseGitDate(fields[3])
ret.CommitDate, _ = ParseGitDate(fields[4])
ret.AuthorEmail = fields[5]
ret.AuthorName = fields[6]
ret.CommitterEmail = fields[7]
ret.CommitterName = fields[8]
if len(fields) > 9 {
ret.Subject = strings.TrimRight(fields[9], "\n")
}
return ret, nil
} else {
msg := fmt.Sprintf("Unexpected output from git show: %v", string(out))
return nil, errors.New(msg)
}
}
func GitAndRootDirs() (string, string, error) {
cmd := subprocess.ExecCommand("git", "rev-parse", "--git-dir", "--show-toplevel")
buf := &bytes.Buffer{}
cmd.Stderr = buf
out, err := cmd.Output()
output := string(out)
if err != nil {
return "", "", fmt.Errorf("Failed to call git rev-parse --git-dir --show-toplevel: %q", buf.String())
}
paths := strings.Split(output, "\n")
pathLen := len(paths)
if pathLen == 0 {
return "", "", fmt.Errorf("Bad git rev-parse output: %q", output)
}
absGitDir, err := filepath.Abs(paths[0])
if err != nil {
return "", "", fmt.Errorf("Error converting %q to absolute: %s", paths[0], err)
}
if pathLen == 1 || len(paths[1]) == 0 {
return absGitDir, "", nil
}
absRootDir, err := filepath.Abs(paths[1])
if err != nil {
return "", "", fmt.Errorf("Error converting %q to absolute: %s", paths[1], err)
}
return absGitDir, absRootDir, nil
}
// Refs returns all of the local and remote branches and tags for the current
// repository. Other refs (HEAD, refs/stash, git notes) are ignored.
func LocalRefs() ([]*Ref, error) {
cmd := subprocess.ExecCommand("git", "show-ref", "--heads", "--tags")
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git show-ref: %v", err)
}
cmd.Start()
var refs []*Ref
scanner := bufio.NewScanner(outp)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
parts := strings.SplitN(line, " ", 2)
if len(parts) != 2 || len(parts[0]) != 40 || len(parts[1]) < 1 {
tracerx.Printf("Invalid line from git show-ref: %q", line)
continue
}
rtype, name := ParseRefToTypeAndName(parts[1])
if rtype != RefTypeLocalBranch && rtype != RefTypeLocalTag {
continue
}
refs = append(refs, &Ref{name, rtype, parts[0]})
}
return refs, cmd.Wait()
}
// GetTrackedFiles returns a list of files which are tracked in Git which match
// the pattern specified (standard wildcard form)
// Both pattern and the results are relative to the current working directory, not
// the root of the repository
func GetTrackedFiles(pattern string) ([]string, error) {
safePattern := sanitizePattern(pattern)
sanitized := len(safePattern) < len(pattern)
var ret []string
cmd := subprocess.ExecCommand("git",
"-c", "core.quotepath=false", // handle special chars in filenames
"ls-files",
"--cached", // include things which are staged but not committed right now
"--", // no ambiguous patterns
safePattern)
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git ls-files: %v", err)
}
cmd.Start()
scanner := bufio.NewScanner(outp)
for scanner.Scan() {
line := scanner.Text()
// If the given pattern was sanitized, then skip all files which
// are not direct cendantsof the repository's root.
if sanitized && filepath.Dir(line) != "." {
continue
}
ret = append(ret, strings.TrimSpace(line))
}
return ret, cmd.Wait()
}
func appendRootCAsFromKeychain(pool *x509.CertPool, name, keychain string) *x509.CertPool {
cmd := subprocess.ExecCommand("/usr/bin/security", "find-certificate", "-a", "-p", "-c", name, keychain)
data, err := cmd.Output()
if err != nil {
tracerx.Printf("Error reading keychain %q: %v", keychain, err)
return pool
}
return appendCertsFromPEMData(pool, data)
}
// RecentBranches returns branches with commit dates on or after the given date/time
// Return full Ref type for easier detection of duplicate SHAs etc
// since: refs with commits on or after this date will be included
// includeRemoteBranches: true to include refs on remote branches
// onlyRemote: set to non-blank to only include remote branches on a single remote
func RecentBranches(since time.Time, includeRemoteBranches bool, onlyRemote string) ([]*Ref, error) {
cmd := subprocess.ExecCommand("git", "for-each-ref",
`--sort=-committerdate`,
`--format=%(refname) %(objectname) %(committerdate:iso)`,
"refs")
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git for-each-ref: %v", err)
}
cmd.Start()
defer cmd.Wait()
scanner := bufio.NewScanner(outp)
// Output is like this:
// refs/heads/master f03686b324b29ff480591745dbfbbfa5e5ac1bd5 2015-08-19 16:50:37 +0100
// refs/remotes/origin/master ad3b29b773e46ad6870fdf08796c33d97190fe93 2015-08-13 16:50:37 +0100
// Output is ordered by latest commit date first, so we can stop at the threshold
regex := regexp.MustCompile(`^(refs/[^/]+/\S+)\s+([0-9A-Za-z]{40})\s+(\d{4}-\d{2}-\d{2}\s+\d{2}\:\d{2}\:\d{2}\s+[\+\-]\d{4})`)
tracerx.Printf("RECENT: Getting refs >= %v", since)
var ret []*Ref
for scanner.Scan() {
line := scanner.Text()
if match := regex.FindStringSubmatch(line); match != nil {
fullref := match[1]
sha := match[2]
reftype, ref := ParseRefToTypeAndName(fullref)
if reftype == RefTypeRemoteBranch || reftype == RefTypeRemoteTag {
if !includeRemoteBranches {
continue
}
if onlyRemote != "" && !strings.HasPrefix(ref, onlyRemote+"/") {
continue
}
}
// This is a ref we might use
// Check the date
commitDate, err := ParseGitDate(match[3])
if err != nil {
return ret, err
}
if commitDate.Before(since) {
// the end
break
}
tracerx.Printf("RECENT: %v (%v)", ref, commitDate)
ret = append(ret, &Ref{ref, reftype, sha})
}
}
return ret, nil
}
func GitDir() (string, error) {
cmd := subprocess.ExecCommand("git", "rev-parse", "--git-dir")
out, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("Failed to call git rev-parse --git-dir: %v %v", err, string(out))
}
path := strings.TrimSpace(string(out))
if len(path) > 0 {
return filepath.Abs(path)
}
return "", nil
}
func RemoteList() ([]string, error) {
cmd := subprocess.ExecCommand("git", "remote")
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git remote: %v", err)
}
cmd.Start()
defer cmd.Wait()
scanner := bufio.NewScanner(outp)
var ret []string
for scanner.Scan() {
ret = append(ret, strings.TrimSpace(scanner.Text()))
}
return ret, nil
}
func appendRootCAsForHostFromPlatform(pool *x509.CertPool, host string) *x509.CertPool {
// Go loads only the system root certificates by default
// see https://github.com/golang/go/blob/master/src/crypto/x509/root_darwin.go
// We want to load certs configured in the System keychain too, this is separate
// from the system root certificates. It's also where other tools such as
// browsers (e.g. Chrome) will load custom trusted certs from. They often
// don't load certs from the login keychain so that's not included here
// either, for consistency.
// find system.keychain for user-added certs (don't assume location)
cmd := subprocess.ExecCommand("/usr/bin/security", "list-keychains")
kcout, err := cmd.Output()
if err != nil {
tracerx.Printf("Error listing keychains: %v", err)
return nil
}
var systemKeychain string
keychains := strings.Split(string(kcout), "\n")
for _, keychain := range keychains {
lc := strings.ToLower(keychain)
if !strings.Contains(lc, "/system.keychain") {
continue
}
systemKeychain = strings.Trim(keychain, " \t\"")
break
}
if len(systemKeychain) == 0 {
return nil
}
pool = appendRootCAsFromKeychain(pool, host, systemKeychain)
// Also check host without port
portreg := regexp.MustCompile(`([^:]+):\d+`)
if match := portreg.FindStringSubmatch(host); match != nil {
hostwithoutport := match[1]
pool = appendRootCAsFromKeychain(pool, hostwithoutport, systemKeychain)
}
return pool
}
func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {
// Start a process per worker
// If concurrent = false we have already dialled back workers to 1
tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
cmd := subprocess.ExecCommand(a.path, a.args)
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
}
inp, err := cmd.StdinPipe()
if err != nil {
return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
}
// Capture stderr to trace
tracer := &traceWriter{}
tracer.processName = filepath.Base(a.path)
cmd.Stderr = tracer
err = cmd.Start()
if err != nil {
return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
}
// Set up buffered reader/writer since we operate on lines
ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer}
// send initiate message
initReq := NewCustomAdapterInitRequest(a.getOperationName(), a.concurrent, a.originalConcurrency)
resp, err := a.exchangeMessage(ctx, initReq)
if err != nil {
a.abortWorkerProcess(ctx)
return nil, err
}
if resp.Error != nil {
a.abortWorkerProcess(ctx)
return nil, fmt.Errorf("Error initializing custom adapter %q worker %d: %v", a.name, workerNum, resp.Error)
}
tracerx.Printf("xfer: started custom adapter process %q for worker %d OK", a.path, workerNum)
// Save this process context and use in future callbacks
return ctx, nil
}
// GetTrackedFiles returns a list of files which are tracked in Git which match
// the pattern specified (standard wildcard form)
// Both pattern and the results are relative to the current working directory, not
// the root of the repository
func GetTrackedFiles(pattern string) ([]string, error) {
var ret []string
cmd := subprocess.ExecCommand("git",
"-c", "core.quotepath=false", // handle special chars in filenames
"ls-files",
"--cached", // include things which are staged but not committed right now
"--", // no ambiguous patterns
pattern)
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git ls-files: %v", err)
}
cmd.Start()
scanner := bufio.NewScanner(outp)
for scanner.Scan() {
line := scanner.Text()
ret = append(ret, strings.TrimSpace(line))
}
return ret, cmd.Wait()
}
func postCloneSubmodules(args []string) error {
// In git 2.9+ the filter option will have been passed through to submodules
// So we need to lfs pull inside each
if !git.Config.IsGitVersionAtLeast("2.9.0") {
// In earlier versions submodules would have used smudge filter
return nil
}
// Also we only do this if --recursive or --recurse-submodules was provided
if !cloneFlags.Recursive && !cloneFlags.RecurseSubmodules {
return nil
}
// Use `git submodule foreach --recursive` to cascade into nested submodules
// Also good to call a new instance of git-lfs rather than do things
// inside this instance, since that way we get a clean env in that subrepo
cmd := subprocess.ExecCommand("git", "submodule", "foreach", "--recursive",
"git lfs pull")
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
return cmd.Run()
}
// CloneWithoutFilters clones a git repo but without the smudge filter enabled
// so that files in the working copy will be pointers and not real LFS data
func CloneWithoutFilters(args []string) error {
// Before git 2.2, setting filters to blank fails, so use cat instead (slightly slower)
filterOverride := ""
if !Config.IsGitVersionAtLeast("2.2.0") {
filterOverride = "cat"
}
// Disable the LFS filters while cloning to speed things up
// this is especially effective on Windows where even calling git-lfs at all
// with --skip-smudge is costly across many files in a checkout
cmdargs := []string{
"-c", fmt.Sprintf("filter.lfs.smudge=%v", filterOverride),
"-c", "filter.lfs.required=false",
"clone"}
cmdargs = append(cmdargs, args...)
cmd := subprocess.ExecCommand("git", cmdargs...)
// Assign pty/tty so git thinks it's a real terminal
tty := subprocess.NewTty(cmd)
stdout, err := tty.Stdout()
if err != nil {
return fmt.Errorf("Failed to get stdout: %v", err)
}
stderr, err := tty.Stderr()
if err != nil {
return fmt.Errorf("Failed to get stderr: %v", err)
}
var outputWait sync.WaitGroup
outputWait.Add(2)
go func() {
io.Copy(os.Stdout, stdout)
outputWait.Done()
}()
go func() {
// Filter stderr to exclude messages caused by disabling the filters
// As of git 2.7 it still tries to call the blank filter but required=false
// this problem should be gone in git 2.8 https://github.com/git/git/commit/1a8630d
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
s := scanner.Text()
// Swallow all the known messages from intentionally breaking filter
if strings.Contains(s, "error: external filter") ||
strings.Contains(s, "error: cannot fork") ||
// Linux / Mac messages
strings.Contains(s, "error: cannot run : No such file or directory") ||
strings.Contains(s, "warning: Clone succeeded, but checkout failed") ||
strings.Contains(s, "You can inspect what was checked out with 'git status'") ||
strings.Contains(s, "retry the checkout") ||
// Windows messages
strings.Contains(s, "error: cannot spawn : No such file or directory") ||
// blank formatting
len(strings.TrimSpace(s)) == 0 {
// Send filtered stderr to trace in case useful
tracerx.Printf(s)
continue
}
os.Stderr.WriteString(s)
os.Stderr.WriteString("\n") // stripped by scanner
}
outputWait.Done()
}()
err = cmd.Start()
if err != nil {
return fmt.Errorf("Failed to start git clone: %v", err)
}
tty.Close()
err = cmd.Wait()
outputWait.Wait()
if err != nil {
return fmt.Errorf("git clone failed: %v", err)
}
return nil
}
// CloneWithoutFilters clones a git repo but without the smudge filter enabled
// so that files in the working copy will be pointers and not real LFS data
func CloneWithoutFilters(flags CloneFlags, args []string) error {
// Before git 2.8, setting filters to blank causes lots of warnings, so use cat instead (slightly slower)
// Also pre 2.2 it failed completely. We used to use it anyway in git 2.2-2.7 and
// suppress the messages in stderr, but doing that with standard StderrPipe suppresses
// the git clone output (git thinks it's not a terminal) and makes it look like it's
// not working. You can get around that with https://github.com/kr/pty but that
// causes difficult issues with passing through Stdin for login prompts
// This way is simpler & more practical.
filterOverride := ""
if !Config.IsGitVersionAtLeast("2.8.0") {
filterOverride = "cat"
}
// Disable the LFS filters while cloning to speed things up
// this is especially effective on Windows where even calling git-lfs at all
// with --skip-smudge is costly across many files in a checkout
cmdargs := []string{
"-c", fmt.Sprintf("filter.lfs.smudge=%v", filterOverride),
"-c", "filter.lfs.required=false",
"clone"}
// flags
if flags.Bare {
cmdargs = append(cmdargs, "--bare")
}
if len(flags.Branch) > 0 {
cmdargs = append(cmdargs, "--branch", flags.Branch)
}
if len(flags.Config) > 0 {
cmdargs = append(cmdargs, "--config", flags.Config)
}
if len(flags.Depth) > 0 {
cmdargs = append(cmdargs, "--depth", flags.Depth)
}
if flags.Dissociate {
cmdargs = append(cmdargs, "--dissociate")
}
if flags.Ipv4 {
cmdargs = append(cmdargs, "--ipv4")
}
if flags.Ipv6 {
cmdargs = append(cmdargs, "--ipv6")
}
if flags.Local {
cmdargs = append(cmdargs, "--local")
}
if flags.Mirror {
cmdargs = append(cmdargs, "--mirror")
}
if flags.NoCheckout {
cmdargs = append(cmdargs, "--no-checkout")
}
if flags.NoHardlinks {
cmdargs = append(cmdargs, "--no-hardlinks")
}
if flags.NoSingleBranch {
cmdargs = append(cmdargs, "--no-single-branch")
}
if len(flags.Origin) > 0 {
cmdargs = append(cmdargs, "--origin", flags.Origin)
}
if flags.Progress {
cmdargs = append(cmdargs, "--progress")
}
if flags.Quiet {
cmdargs = append(cmdargs, "--quiet")
}
if flags.Recursive {
cmdargs = append(cmdargs, "--recursive")
}
if flags.RecurseSubmodules {
cmdargs = append(cmdargs, "--recurse-submodules")
}
if len(flags.Reference) > 0 {
cmdargs = append(cmdargs, "--reference", flags.Reference)
}
if len(flags.SeparateGit) > 0 {
cmdargs = append(cmdargs, "--separate-git-dir", flags.SeparateGit)
}
if flags.Shared {
cmdargs = append(cmdargs, "--shared")
}
if flags.SingleBranch {
cmdargs = append(cmdargs, "--single-branch")
}
if len(flags.TemplateDirectory) > 0 {
cmdargs = append(cmdargs, "--template", flags.TemplateDirectory)
}
if len(flags.Upload) > 0 {
cmdargs = append(cmdargs, "--upload-pack", flags.Upload)
}
if flags.Verbose {
cmdargs = append(cmdargs, "--verbose")
}
// Now args
cmdargs = append(cmdargs, args...)
cmd := subprocess.ExecCommand("git", cmdargs...)
// Assign all streams direct
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
err := cmd.Start()
if err != nil {
return fmt.Errorf("Failed to start git clone: %v", err)
}
err = cmd.Wait()
if err != nil {
return fmt.Errorf("git clone failed: %v", err)
}
return nil
}