func (this *VcsCommand) Run(job models.Job) int {
repo := job.RepoLocation
localFolder := job.LocalFolder
if _, err := os.Stat(localFolder); err != nil {
log.Println(err)
}
if err := os.MkdirAll(localFolder, os.ModePerm); err != nil {
log.Println(err)
}
privateFolder := filepath.Join(localFolder, ".git")
cmd := vcs.ByCmd("git")
if _, err := os.Stat(privateFolder); err != nil {
fmt.Printf("Cloning %s ...\n", repo)
cmd.Create(localFolder, repo)
fmt.Printf("Finished cloning %s.\n", repo)
} else {
fmt.Printf("Updating %s ...\n", repo)
cmd.Download(localFolder)
fmt.Printf("Finished updating %s.\n", repo)
}
return 0
}
"golang.org/x/tools/go/vcs"
)
type VCS struct {
vcs *vcs.Cmd
identifyCmd string
describeCmd string
diffCmd string
// run in sandbox repos
existsCmd string
}
var vcsBzr = &VCS{
vcs: vcs.ByCmd("bzr"),
identifyCmd: "version-info --custom --template {revision_id}",
describeCmd: "revno", // TODO(kr): find tag names if possible
diffCmd: "diff -r {rev}",
}
var vcsGit = &VCS{
vcs: vcs.ByCmd("git"),
identifyCmd: "rev-parse HEAD",
describeCmd: "describe --tags",
diffCmd: "diff {rev}",
existsCmd: "cat-file -e {rev}",
}
// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
gccpath := filepath.Join(repo.Path, "gcc")
// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
git := vcs.ByCmd("git")
// only pull down gcc if we don't have a local copy.
if _, err := os.Stat(gccpath); err != nil {
if err := timeout(*cmdTimeout, func() error {
// pull down a working copy of GCC.
return git.Create(gccpath, *gccPath)
}); err != nil {
return "", err
}
}
if err := git.Download(gccpath); err != nil {
return "", err
}
// get the modified files for this commit.
var buf bytes.Buffer
if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
}
modifiedFiles := strings.Split(buf.String(), "\n")
var isMirrored bool
for _, f := range modifiedFiles {
if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
isMirrored = true
break
}
}
// use git log to find the corresponding commit to sync to in the gcc mirror.
// If the files modified in the gofrontend are mirrored to gcc, we expect a
// commit with a similar description in the gcc mirror. If the files modified are
// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
// occurred before those files were modified to verify gccgo's status at that point.
logCmd := []string{
"log",
"-1",
"--format=%H",
}
var errMsg string
if isMirrored {
commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
if err != nil {
return "", err
}
quotedDesc := regexp.QuoteMeta(string(commitDesc))
logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
} else {
commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
if err != nil {
return "", err
}
logCmd = append(logCmd, "--before", string(commitDate))
errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
}
buf.Reset()
if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
return "", fmt.Errorf("%s: %s", errMsg, err)
}
gccRev := buf.String()
if gccRev == "" {
return "", fmt.Errorf(errMsg)
}
// checkout gccRev
// TODO(cmang): Fix this to work in parallel mode.
if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
}
// make objdir to work in
gccobjdir := filepath.Join(workpath, "gcc-objdir")
if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
return "", err
}
// configure GCC with substituted gofrontend and libgo
if err := run(exec.Command(filepath.Join(gccpath, "configure"),
"--enable-languages=c,c++,go",
"--disable-bootstrap",
"--disable-multilib",
), runEnv(envv), runDir(gccobjdir)); err != nil {
return "", fmt.Errorf("Failed to configure GCC: %v", err)
}
// build gcc
if err := run(exec.Command("make"), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
return "", fmt.Errorf("Failed to build GCC: %s", err)
}
return gccobjdir, nil
}
// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
gccpath := filepath.Join(repo.Path, "gcc")
// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
git := vcs.ByCmd("git")
// only pull down gcc if we don't have a local copy.
if _, err := os.Stat(gccpath); err != nil {
if err := timeout(*cmdTimeout, func() error {
// pull down a working copy of GCC.
cloneCmd := []string{
"clone",
// This is just a guess since there are ~6000 commits to
// GCC per year. It's likely there will be enough history
// to cross-reference the Gofrontend commit against GCC.
// The disadvantage would be if the commit being built is more than
// a year old; in this case, the user should make a clone that has
// the full history.
"--depth", "6000",
// We only care about the master branch.
"--branch", "master", "--single-branch",
*gccPath,
}
// Clone Kind Clone Time(Dry run) Clone Size
// ---------------------------------------------------------------
// Full Clone 10 - 15 min 2.2 GiB
// Master Branch 2 - 3 min 1.5 GiB
// Full Clone(shallow) 1 min 900 MiB
// Master Branch(shallow) 40 sec 900 MiB
//
// The shallow clones have the same size, which is expected,
// but the full shallow clone will only have 6000 commits
// spread across all branches. There are ~50 branches.
return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
}); err != nil {
return "", err
}
}
if err := git.Download(gccpath); err != nil {
return "", err
}
// get the modified files for this commit.
var buf bytes.Buffer
if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
}
modifiedFiles := strings.Split(buf.String(), "\n")
var isMirrored bool
for _, f := range modifiedFiles {
if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
isMirrored = true
break
}
}
// use git log to find the corresponding commit to sync to in the gcc mirror.
// If the files modified in the gofrontend are mirrored to gcc, we expect a
// commit with a similar description in the gcc mirror. If the files modified are
// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
// occurred before those files were modified to verify gccgo's status at that point.
logCmd := []string{
"log",
"-1",
"--format=%H",
}
var errMsg string
if isMirrored {
commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
if err != nil {
return "", err
}
quotedDesc := regexp.QuoteMeta(string(commitDesc))
logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
} else {
commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
if err != nil {
return "", err
}
logCmd = append(logCmd, "--before", string(commitDate))
errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
}
buf.Reset()
if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
return "", fmt.Errorf("%s: %s", errMsg, err)
}
gccRev := buf.String()
if gccRev == "" {
return "", fmt.Errorf(errMsg)
}
// checkout gccRev
// TODO(cmang): Fix this to work in parallel mode.
if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
}
// make objdir to work in
gccobjdir := filepath.Join(workpath, "gcc-objdir")
if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
return "", err
}
// configure GCC with substituted gofrontend and libgo
if err := run(exec.Command(filepath.Join(gccpath, "configure"),
"--enable-languages=c,c++,go",
"--disable-bootstrap",
"--disable-multilib",
), runEnv(envv), runDir(gccobjdir)); err != nil {
return "", fmt.Errorf("Failed to configure GCC: %v", err)
}
// build gcc
if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
return "", fmt.Errorf("Failed to build GCC: %s", err)
}
return gccobjdir, nil
}
return true, res, err
}
return false, res, err
}
// VCSType represents a prefix to look for, a scheme to ping a path
// with and a VCS command to do the pinging.
type VCSType struct {
Prefix string
Scheme string
VCS *vcs.Cmd
}
// VCSTypes is the list of VCSType used by GuessVCS
var VCSTypes = []VCSType{
{"git+ssh://", "git+ssh", vcs.ByCmd("git")},
{"git://", "git", vcs.ByCmd("git")},
{"git@", "git", GitAtVCS()},
{"ssh://hg@", "ssh", vcs.ByCmd("hg")},
{"svn://", "svn", vcs.ByCmd("svn")},
{"bzr://", "bzr", vcs.ByCmd("bzr")},
{"https://", "https", vcs.ByCmd("git")}, // not so sure this is a good idea
}
// GuessVCS attempts to guess the VCS given a url. This uses the
// VCSTypes array, checking for prefixes that match and attempting to
// ping the VCS with the given scheme
func GuessVCS(url string) *vcs.Cmd {
for _, vt := range VCSTypes {
if !strings.HasPrefix(url, vt.Prefix) {
continue