Ejemplo n.º 1
0
func (this *VcsCommand) Run(job models.Job) int {

	repo := job.RepoLocation
	localFolder := job.LocalFolder

	if _, err := os.Stat(localFolder); err != nil {
		log.Println(err)
	}

	if err := os.MkdirAll(localFolder, os.ModePerm); err != nil {
		log.Println(err)
	}

	privateFolder := filepath.Join(localFolder, ".git")

	cmd := vcs.ByCmd("git")

	if _, err := os.Stat(privateFolder); err != nil {
		fmt.Printf("Cloning %s ...\n", repo)
		cmd.Create(localFolder, repo)
		fmt.Printf("Finished cloning %s.\n", repo)
	} else {
		fmt.Printf("Updating %s ...\n", repo)
		cmd.Download(localFolder)
		fmt.Printf("Finished updating %s.\n", repo)
	}

	return 0
}
Ejemplo n.º 2
0
	"golang.org/x/tools/go/vcs"
)

type VCS struct {
	vcs *vcs.Cmd

	identifyCmd string
	describeCmd string
	diffCmd     string

	// run in sandbox repos
	existsCmd string
}

var vcsBzr = &VCS{
	vcs: vcs.ByCmd("bzr"),

	identifyCmd: "version-info --custom --template {revision_id}",
	describeCmd: "revno", // TODO(kr): find tag names if possible
	diffCmd:     "diff -r {rev}",
}

var vcsGit = &VCS{
	vcs: vcs.ByCmd("git"),

	identifyCmd: "rev-parse HEAD",
	describeCmd: "describe --tags",
	diffCmd:     "diff {rev}",

	existsCmd: "cat-file -e {rev}",
}
Ejemplo n.º 3
0
// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.
			return git.Create(gccpath, *gccPath)
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make"), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}
Ejemplo n.º 4
0
// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.

			cloneCmd := []string{
				"clone",
				// This is just a guess since there are ~6000 commits to
				// GCC per year. It's likely there will be enough history
				// to cross-reference the Gofrontend commit against GCC.
				// The disadvantage would be if the commit being built is more than
				// a year old; in this case, the user should make a clone that has
				// the full history.
				"--depth", "6000",
				// We only care about the master branch.
				"--branch", "master", "--single-branch",
				*gccPath,
			}

			// Clone Kind			Clone Time(Dry run)	Clone Size
			// ---------------------------------------------------------------
			// Full Clone			10 - 15 min		2.2 GiB
			// Master Branch		2 - 3 min		1.5 GiB
			// Full Clone(shallow)		1 min			900 MiB
			// Master Branch(shallow)	40 sec			900 MiB
			//
			// The shallow clones have the same size, which is expected,
			// but the full shallow clone will only have 6000 commits
			// spread across all branches.  There are ~50 branches.
			return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}
Ejemplo n.º 5
0
		return true, res, err
	}
	return false, res, err
}

// VCSType represents a prefix to look for, a scheme to ping a path
// with and a VCS command to do the pinging.
type VCSType struct {
	Prefix string
	Scheme string
	VCS    *vcs.Cmd
}

// VCSTypes is the list of VCSType used by GuessVCS
var VCSTypes = []VCSType{
	{"git+ssh://", "git+ssh", vcs.ByCmd("git")},
	{"git://", "git", vcs.ByCmd("git")},
	{"git@", "git", GitAtVCS()},
	{"ssh://hg@", "ssh", vcs.ByCmd("hg")},
	{"svn://", "svn", vcs.ByCmd("svn")},
	{"bzr://", "bzr", vcs.ByCmd("bzr")},
	{"https://", "https", vcs.ByCmd("git")}, // not so sure this is a good idea
}

// GuessVCS attempts to guess the VCS given a url. This uses the
// VCSTypes array, checking for prefixes that match and attempting to
// ping the VCS with the given scheme
func GuessVCS(url string) *vcs.Cmd {
	for _, vt := range VCSTypes {
		if !strings.HasPrefix(url, vt.Prefix) {
			continue