// repoForTool returns the correct RepoRoot for the buildTool, or an error if // the tool is unknown. func repoForTool() (*vcs.RepoRoot, error) { switch *buildTool { case "go": return vcs.RepoRootForImportPath(*gcPath, *verbose) case "gccgo": return vcs.RepoRootForImportPath(gofrontendImportPath, *verbose) default: return nil, fmt.Errorf("unknown build tool: %s", *buildTool) } }
// Behind takes a github token and a godep file // and returns a list of dependencies and if they are out of date func Behind(githubToken string, godepFile string) []ImportStatus { gh := NewGitHub(githubToken) gd, err := LoadGodepsFile(godepFile) if err != nil { log.Fatalf("Error loading godeps file %q: %s", godepFile, err) } roots := make(map[string]bool, len(gd.Deps)) imports := make([]ImportStatus, 0, len(gd.Deps)) for _, dep := range gd.Deps { rr, err := vcs.RepoRootForImportPath(dep.ImportPath, false) if err != nil { log.Printf("Unable to process %s: %s", dep.ImportPath, err) continue } if roots[rr.Root] { continue } roots[rr.Root] = true parts := strings.Split(dep.ImportPath, "/") if len(parts) < 2 { log.Printf("Skipping %s", dep.ImportPath) continue } if parts[0] == "golang.org" && parts[1] == "x" { parts[0] = "github.com" parts[1] = "golang" } if parts[0] != "github.com" { log.Printf("Skipping %s", dep.ImportPath) continue } compare, _, err := gh.Client.Repositories.CompareCommits(parts[1], parts[2], dep.Rev, "HEAD") if err != nil { log.Printf("got error reading repo %s: %s", dep.ImportPath, err) continue } status := ImportStatus{ Root: rr.Root, Status: *compare.Status, Commits: make([]CommitMini, 0, len(compare.Commits)), } for _, c := range compare.Commits { msg := "" if c.Commit.Message != nil { msg = *c.Commit.Message } status.Commits = append(status.Commits, CommitMini{ SHA: *c.SHA, Msg: msg, }) } imports = append(imports, status) } return imports }
func (*workspace) computeVCSState(r *Repo) { if r.vcs == nil { // Go package not under VCS. return } r.DefaultBranch = r.vcs.DefaultBranch() if s, err := r.vcs.Status(r.Path); err == nil { r.Local.Status = s } if b, err := r.vcs.Branch(r.Path); err == nil { r.Local.Branch = b } if rev, err := r.vcs.LocalRevision(r.Path); err == nil { r.Local.Revision = rev } if s, err := r.vcs.Stash(r.Path); err == nil { r.Local.Stash = s } if remote, err := r.vcs.RemoteURL(r.Path); err == nil { r.Local.RemoteURL = remote } if rev, err := r.vcs.RemoteRevision(r.Path); err == nil { r.Remote.Revision = rev } if r.Remote.Revision != "" { if c, err := r.vcs.Contains(r.Path, r.Remote.Revision); err == nil { r.LocalContainsRemoteRevision = c } } if rr, err := vcs.RepoRootForImportPath(r.Root, false); err == nil { r.Remote.RepoURL = rr.Repo } }
// getRepoRoot takes an import path like github.com/sparrc/gdm // and returns the VCS Repository information for it. func getRepoRoot(importpath string) (*vcs.RepoRoot, error) { repo, err := vcs.RepoRootForImportPath(importpath, false) if err != nil { return nil, err } return repo, nil }
func (custom *CmakeCustomizations) createRepo() error { Info(fmt.Sprintf("Downloading %s", custom.Configuration.Project)) repo, err := vcs.RepoRootForImportPath(custom.Configuration.Project, false) if err != nil { return err } return repo.VCS.Create(custom.paths.src, repo.Repo) }
// RemoteRepo constructs a *Repo representing a remote repository. func RemoteRepo(url, path string) (*Repo, error) { rr, err := vcs.RepoRootForImportPath(url, *verbose) if err != nil { return nil, err } return &Repo{ Path: path, Master: rr, }, nil }
func FromImportPath(importPath string) (*VCS, error) { rr, err := vcs.RepoRootForImportPath(importPath, false) if err != nil { return nil, err } vcs := cmd[rr.VCS] if vcs == nil { return nil, fmt.Errorf("%s is unsupported: %s", rr.VCS.Name, importPath) } return vcs, nil }
func (this *GoPackage) UpdateVcsFields() { if this.Dir.Repo == nil { return } gist7802150.MakeUpdated(this.Dir.Repo.VcsLocal) gist7802150.MakeUpdated(this.Dir.Repo.VcsRemote) repoImportPath := GetRepoImportPath(this.Dir.Repo.Vcs.RootPath(), this.Bpkg.SrcRoot) if repoRoot, err := vcs.RepoRootForImportPath(repoImportPath, false); err == nil { this.Dir.Repo.RepoRoot = repoRoot } }
// NewPackage returns new package. func NewPackage(name, gopath string) (*Package, error) { dir := filepath.Join(gopath, "src", name) repo, err := vcs.RepoRootForImportPath(name, false) if err != nil { // it's ok, silently discard errors here return nil, err } return &Package{ Name: name, Dir: dir, Repo: repo, }, nil }
// mergeGodeps will get one master list of revs. func (w *workspace) mergeGodeps(dirGs map[string]Godeps) map[string]dirDep { roots := map[string]dirDep{} for dir, g := range dirGs { for _, dep := range g.Deps { repoRoot, err := vcs.RepoRootForImportPath(dep.ImportPath, false) if err != nil { fmt.Fprintf(os.Stderr, "for %q: %s\n", dep.ImportPath, err) continue } dd := dirDep{ srcDir: dir, rev: dep.Rev, repo: repoRoot.Repo, root: filepath.Join(w.vendorRootSrc(), repoRoot.Root), kind: repoRoot.VCS.Cmd, } if orig, ok := roots[repoRoot.Repo]; ok { if orig != dd { fmt.Fprintf(os.Stderr, "conflict for %q: godeps in %q and %q do not match\n", dd.repo, orig.srcDir, dd.srcDir) continue } } else { roots[repoRoot.Root] = dd } } } // clear out nested var rootDirs sort.StringSlice for _, dd := range roots { rootDirs = append(rootDirs, dd.root) } sort.Sort(rootDirs) var last string for _, r := range rootDirs { if last != "" && strings.HasPrefix(r, last) { delete(roots, r) fmt.Fprintf(os.Stderr, "ignoring %q which is managed in %q\n", r, last) } else { last = r + string(filepath.Separator) } } return roots }
func getSpec(repoName string) (string, error) { repo, err := vcs.RepoRootForImportPath(repoName, true) if err != nil { return "", err } path := filepath.Join(GetQuiltPath(), repo.Root) if _, err := util.AppFs.Stat(path); os.IsNotExist(err) { log.Info(fmt.Sprintf("Cloning %s into %s", repo.Root, path)) if err := create(repo, path); err != nil { return "", err } } else { log.Info(fmt.Sprintf("Updating %s in %s", repo.Root, path)) download(repo, path) } return path, nil }
// commitWatcher polls hg for new commits and tells the dashboard about them. func commitWatcher(goroot *Repo) { if *commitInterval == 0 { log.Printf("commitInterval is 0; disabling commitWatcher") return } if !*report { log.Printf("-report is false; disabling commitWatcher") return } // Create builder just to get master key. b, err := NewBuilder(goroot, "mercurial-commit") if err != nil { log.Fatal(err) } key := b.key benchMutex.RLock() for { if *verbose { log.Printf("poll...") } // Main Go repository. commitPoll(goroot, "", key) // Go sub-repositories. for _, pkg := range dashboardPackages("subrepo") { pkgmaster, err := vcs.RepoRootForImportPath(pkg, *verbose) if err != nil { log.Printf("Error finding subrepo (%s): %s", pkg, err) continue } pkgroot := &Repo{ Path: filepath.Join(*buildroot, pkg), Master: pkgmaster, } commitPoll(pkgroot, pkg, key) } benchMutex.RUnlock() if *verbose { log.Printf("sleep...") } time.Sleep(*commitInterval) benchMutex.RLock() } }
// buildSubrepo fetches the given package, updates it to the specified hash, // and runs 'go test -short pkg/...'. It returns the build log and any error. func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) { goTool := filepath.Join(goRoot, "bin", "go") + exeExt env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath) // add $GOROOT/bin and $GOPATH/bin to PATH for i, e := range env { const p = "PATH=" if !strings.HasPrefix(e, p) { continue } sep := string(os.PathListSeparator) env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):] } // HACK: check out to new sub-repo location instead of old location. pkg = strings.Replace(pkg, "code.google.com/p/go.", "golang.org/x/", 1) // fetch package and dependencies var outbuf bytes.Buffer err := run(exec.Command(goTool, "get", "-d", pkg+"/..."), runEnv(env), allOutput(&outbuf), runDir(goPath)) if err != nil { return outbuf.String(), err } outbuf.Reset() // hg update to the specified hash pkgmaster, err := vcs.RepoRootForImportPath(pkg, *verbose) if err != nil { return "", fmt.Errorf("Error finding subrepo (%s): %s", pkg, err) } repo := &Repo{ Path: filepath.Join(goPath, "src", pkg), Master: pkgmaster, } if err := repo.UpdateTo(hash); err != nil { return "", err } // test the package err = run(exec.Command(goTool, "test", "-short", pkg+"/..."), runTimeout(*buildTimeout), runEnv(env), allOutput(&outbuf), runDir(goPath)) return outbuf.String(), err }
// ResolveRepo on a default reporesolver is effectively go get wraped // to use the url string. func (dr *DefaultRepoResolver) ResolveRepo(importPath string, dep *CanticleDependency) (VCS, error) { // We guess our vcs based off our url path if present resolvePath := getResolvePath(importPath) LogVerbose("Attempting to use go get vcs for url: %s", resolvePath) vcs.Verbose = Verbose repo, err := vcs.RepoRootForImportPath(resolvePath, true) if err != nil { LogVerbose("Failed creating VCS for url: %s, err: %s", resolvePath, err.Error()) return nil, err } // If we found something return non nil repo.Root, err = TrimPathToRoot(importPath, repo.Root) if err != nil { LogVerbose("Failed creating VCS for url: %s, err: %s", resolvePath, err.Error()) return nil, err } v := &PackageVCS{Repo: repo, Gopath: dr.Gopath} LogVerbose("Created VCS for url: %s", resolvePath) return v, nil }
// download the given dependency. // 2 Passes: 1) go get -d <pkg>, 2) git pull (if necessary) func download(dep *Dependency) error { rr, err := vcs.RepoRootForImportPath(dep.ImportPath, debug) if err != nil { debugln("Error determining repo root for", dep.ImportPath) return err } ppln("rr", rr) dep.vcs = cmd[rr.VCS] // try to find an existing directory in the GOPATHs for _, gp := range filepath.SplitList(build.Default.GOPATH) { t := filepath.Join(gp, "src", rr.Root) fi, err := os.Stat(t) if err != nil { continue } if fi.IsDir() { dep.root = t break } } // If none found, just pick the first GOPATH entry (AFAICT that's what go get does) if dep.root == "" { dep.root = filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "src", rr.Root) } ppln("dep", dep) if downloaded[rr.Repo] { verboseln("Skipping already downloaded repo", rr.Repo) return nil } fi, err := os.Stat(dep.root) if err != nil { if os.IsNotExist(err) { if err := os.MkdirAll(filepath.Dir(dep.root), os.ModePerm); err != nil { debugln("Error creating base dir of", dep.root) return err } err := rr.VCS.CreateAtRev(dep.root, rr.Repo, dep.Rev) debugln("CreatedAtRev", dep.root, rr.Repo, dep.Rev) if err != nil { debugln("CreateAtRev error", err) return err } downloaded[rr.Repo] = true return nil } debugln("Error checking repo root for", dep.ImportPath, "at", dep.root, ":", err) return err } if !fi.IsDir() { return errors.New("repo root src dir exists, but isn't a directory for " + dep.ImportPath + " at " + dep.root) } if !dep.vcs.exists(dep.root, dep.Rev) { debugln("Updating existing", dep.root) if dep.vcs == vcsGit { detached, err := gitDetached(dep.root) if err != nil { return err } if detached { db, err := gitDefaultBranch(dep.root) if err != nil { return err } if err := gitCheckout(dep.root, db); err != nil { return err } } } dep.vcs.vcs.Download(dep.root) downloaded[rr.Repo] = true } debugln("Nothing to download") return nil }
// TODO: refactor this function into multiple smaller ones. Currently all the // code is in this function only due to the os.RemoveAll(tempdir). func makeUpstreamSourceTarball(gopkg string) (string, string, map[string]bool, string, error) { // dependencies is a map in order to de-duplicate multiple imports // pointing into the same repository. dependencies := make(map[string]bool) autoPkgType := "library" tempdir, err := ioutil.TempDir("", "dh-make-golang") if err != nil { return "", "", dependencies, autoPkgType, err } defer os.RemoveAll(tempdir) log.Printf("Downloading %q\n", gopkg+"/...") done := make(chan bool) go progressSize("go get", filepath.Join(tempdir, "src"), done) // As per https://groups.google.com/forum/#!topic/golang-nuts/N5apfenE4m4, // the arguments to “go get” are packages, not repositories. Hence, we // specify “gopkg/...” in order to cover all packages. // As a concrete example, github.com/jacobsa/util is a repository we want // to package into a single Debian package, and using “go get -d // github.com/jacobsa/util” fails because there are no buildable go files // in the top level of that repository. cmd := exec.Command("go", "get", "-d", "-t", gopkg+"/...") cmd.Stderr = os.Stderr cmd.Env = append([]string{ fmt.Sprintf("GOPATH=%s", tempdir), }, passthroughEnv()...) if err := cmd.Run(); err != nil { done <- true return "", "", dependencies, autoPkgType, err } done <- true fmt.Printf("\r") revision := strings.TrimSpace(*gitRevision) if revision != "" { log.Printf("Checking out git revision %q\n", revision) if err := runGitCommandIn(filepath.Join(tempdir, "src", gopkg), "reset", "--hard", *gitRevision); err != nil { log.Fatalf("Could not check out git revision %q: %v\n", revision, err) } log.Printf("Refreshing %q\n", gopkg+"/...") done := make(chan bool) go progressSize("go get", filepath.Join(tempdir, "src"), done) cmd := exec.Command("go", "get", "-d", "-t", gopkg+"/...") cmd.Stderr = os.Stderr cmd.Env = append([]string{ fmt.Sprintf("GOPATH=%s", tempdir), }, passthroughEnv()...) if err := cmd.Run(); err != nil { done <- true return "", "", dependencies, autoPkgType, err } done <- true fmt.Printf("\r") } if _, err := os.Stat(filepath.Join(tempdir, "src", gopkg, "debian")); err == nil { log.Printf("WARNING: ignoring debian/ directory that came with the upstream sources\n") } f, err := ioutil.TempFile("", "dh-make-golang") tempfile := f.Name() f.Close() base := filepath.Base(gopkg) dir := filepath.Dir(gopkg) cmd = exec.Command( "tar", "cJf", tempfile, "--exclude-vcs", "--exclude=Godeps", "--exclude=vendor", fmt.Sprintf("--exclude=%s/debian", base), base) cmd.Dir = filepath.Join(tempdir, "src", dir) cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return "", "", dependencies, autoPkgType, err } if _, err := os.Stat(filepath.Join(tempdir, "src", gopkg, ".git")); os.IsNotExist(err) { return "", "", dependencies, autoPkgType, fmt.Errorf("Not a git repository, dh-make-golang currently only supports git") } log.Printf("Determining upstream version number\n") version, err := pkgVersionFromGit(filepath.Join(tempdir, "src", gopkg)) if err != nil { return "", "", dependencies, autoPkgType, err } log.Printf("Package version is %q\n", version) // If no import path defines a “main” package, we’re dealing with a // library, otherwise likely a program. log.Printf("Determining package type\n") cmd = exec.Command("go", "list", "-f", "{{.ImportPath}} {{.Name}}", gopkg+"/...") cmd.Stderr = os.Stderr cmd.Env = append([]string{ fmt.Sprintf("GOPATH=%s", tempdir), }, passthroughEnv()...) out, err := cmd.Output() if err != nil { return "", "", dependencies, autoPkgType, err } for _, line := range strings.Split(string(out), "\n") { if strings.TrimSpace(line) == "" { continue } if strings.Contains(line, "/vendor/") || strings.Contains(line, "/Godeps/") || strings.Contains(line, "/samples/") || strings.Contains(line, "/examples/") || strings.Contains(line, "/example/") { continue } if strings.HasSuffix(line, " main") { if strings.TrimSpace(*pkgType) == "" { log.Printf("Assuming you are packaging a program (because %q defines a main package), use -type to override\n", line[:len(line)-len(" main")]) } autoPkgType = "program" break } } log.Printf("Determining dependencies\n") cmd = exec.Command("go", "list", "-f", "{{join .Imports \"\\n\"}}\n{{join .TestImports \"\\n\"}}\n{{join .XTestImports \"\\n\"}}", gopkg+"/...") cmd.Stderr = os.Stderr cmd.Env = append([]string{ fmt.Sprintf("GOPATH=%s", tempdir), }, passthroughEnv()...) out, err = cmd.Output() if err != nil { return "", "", dependencies, autoPkgType, err } var godependencies []string for _, p := range strings.Split(string(out), "\n") { if strings.TrimSpace(p) == "" { continue } // Strip packages that are included in the repository we are packaging. if strings.HasPrefix(p, gopkg) { continue } if p == "C" { // TODO: maybe parse the comments to figure out C deps from pkg-config files? } else { godependencies = append(godependencies, p) } } if len(godependencies) == 0 { return tempfile, version, dependencies, autoPkgType, nil } // Remove all packages which are in the standard lib. args := []string{"list", "-f", "{{.ImportPath}}: {{.Standard}}"} args = append(args, godependencies...) cmd = exec.Command("go", args...) cmd.Dir = filepath.Join(tempdir, "src", gopkg) cmd.Stderr = os.Stderr cmd.Env = append([]string{ fmt.Sprintf("GOPATH=%s", tempdir), }, passthroughEnv()...) out, err = cmd.Output() if err != nil { return "", "", dependencies, autoPkgType, err } for _, p := range strings.Split(string(out), "\n") { if strings.HasSuffix(p, ": false") { importpath := p[:len(p)-len(": false")] rr, err := vcs.RepoRootForImportPath(importpath, false) if err != nil { log.Printf("Could not determine repo path for import path %q: %v\n", importpath, err) } dependencies[debianNameFromGopkg(rr.Root, "library")+"-dev"] = true } } return tempfile, version, dependencies, autoPkgType, nil }
func (p *Plug) fetchRoot() (*vcs.RepoRoot, error) { return vcs.RepoRootForImportPath(p.Repo, false) }
// LoadDependencies is not done func LoadDependencies(pkgs []string, ignores []string) ([]Dependency, error) { stdlib, err := golist.Std() if err != nil { return nil, err } pkgs, err = golist.Deps(pkgs...) if err != nil { return nil, err } // faster to remove stdlib pkgs = removeIfEquals(pkgs, stdlib) pkgs = removeIfSubstring(pkgs, ignores) deps, err := golist.Packages(pkgs...) if err != nil { return nil, err } visited := make(map[string]string, len(deps)) out := make([]Dependency, 0, len(deps)) for _, v := range deps { src := filepath.Join(v.Root, "src") path := filepath.Join(src, filepath.FromSlash(v.ImportPath)) cmd, _, err := vcs.FromDir(path, src) if err != nil { log.Printf("error computing vcs %s: %s", path, err) continue } rr, err := vcs.RepoRootForImportPath(v.ImportPath, false) if err != nil { log.Printf("error computing repo for %s: %s", v.ImportPath, err) continue } e := Dependency{ Package: v, } e.Project.Repo = rr.Repo e.Project.VcsName = cmd.Name e.Project.VcsCmd = cmd.Cmd e.License = GetLicense(path) visited[v.ImportPath] = e.License.Type // if child have no license, parent has license, continue // if child has no license, parent has no license, carry on if e.License.Type == "" { lic, ok := visited[rr.Root] if ok && lic != "" { // case is child has no license, but parent does // => just ignore this package continue } // if ok && lic = "" => dont look up parent if !ok { // first time checking parent parentpkg, err := golist.GetPackage(rr.Root) if err == nil { parent := Dependency{ Package: parentpkg, Project: e.Project, } path = filepath.Join(src, filepath.FromSlash(rr.Root)) parent.License = GetLicense(path) visited[rr.Root] = parent.License.Type if parent.License.Type != "" { // case child has no license, parent does // ignore child and use parent e = parent } } } } commit, err := GetLastCommit(path) if err == nil { e.Commit = commit } e.Project.LicenseLink = LinkToFile(e.ImportPath, e.License.File, e.Commit.Rev) out = append(out, e) } return out, err }
// CheckHandler handles the request for checking a repo func CheckHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") repo := r.FormValue("repo") repoRoot, err := vcs.RepoRootForImportPath(repo, true) if err != nil || repoRoot.Root == "" || repoRoot.Repo == "" { log.Println("Failed to create repoRoot:", repoRoot, err) b, marshalErr := json.Marshal("Please enter a valid 'go get'-able package name") if marshalErr != nil { log.Println("JSON marshal error:", marshalErr) } w.WriteHeader(http.StatusBadRequest) w.Write(b) return } log.Printf("Checking repo %q...", repo) forceRefresh := r.Method != "GET" // if this is a GET request, try to fetch from cached version in boltdb first resp, err := newChecksResp(repo, forceRefresh) if err != nil { log.Println("ERROR: from newChecksResp:", err) b, marshalErr := json.Marshal("Could not go get the repository.") if marshalErr != nil { log.Println("JSON marshal error:", marshalErr) } w.WriteHeader(http.StatusBadRequest) w.Write(b) return } respBytes, err := json.Marshal(resp) if err != nil { log.Println("ERROR: could not marshal json:", err) http.Error(w, err.Error(), 500) return } w.Write(respBytes) // write to boltdb db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { log.Println("Failed to open bolt database: ", err) return } defer db.Close() // is this a new repo? if so, increase the count in the high scores bucket later isNewRepo := false err = db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(RepoBucket)) if b == nil { return fmt.Errorf("repo bucket not found") } isNewRepo = b.Get([]byte(repo)) == nil return nil }) if err != nil { log.Println(err) } // if this is a new repo, or the user force-refreshed, update the cache if isNewRepo || forceRefresh { err = db.Update(func(tx *bolt.Tx) error { log.Printf("Saving repo %q to cache...", repo) b := tx.Bucket([]byte(RepoBucket)) if b == nil { return fmt.Errorf("repo bucket not found") } // save repo to cache err = b.Put([]byte(repo), respBytes) if err != nil { return err } // fetch meta-bucket mb := tx.Bucket([]byte(MetaBucket)) if mb == nil { return fmt.Errorf("high score bucket not found") } // update total repos count if isNewRepo { err = updateReposCount(mb, resp, repo) if err != nil { return err } } return updateHighScores(mb, resp, repo) }) if err != nil { log.Println("Bolt writing error:", err) } } return }