// From a local repo find out the current branch name if there is one.
func findCurrentBranch(repo v.Repo) string {
msg.Debug("Attempting to find current branch for %s", repo.Remote())
// Svn and Bzr don't have default branches.
if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr {
return ""
}
if repo.Vcs() == v.Git {
c := exec.Command("git", "symbolic-ref", "--short", "HEAD")
c.Dir = repo.LocalPath()
c.Env = envForDir(c.Dir)
out, err := c.CombinedOutput()
if err != nil {
msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err)
return ""
}
return strings.TrimSpace(string(out))
}
if repo.Vcs() == v.Hg {
c := exec.Command("hg", "branch")
c.Dir = repo.LocalPath()
c.Env = envForDir(c.Dir)
out, err := c.CombinedOutput()
if err != nil {
msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err)
return ""
}
return strings.TrimSpace(string(out))
}
return ""
}
// NotFound attempts to retrieve a package when not found in the local vendor/
// folder. It will attempt to get it from the remote location info.
func (m *MissingPackageHandler) NotFound(pkg string, addTest bool) (bool, error) {
root := util.GetRootFromPackage(pkg)
// Skip any references to the root package.
if root == m.Config.Name {
return false, nil
}
dest := filepath.Join(m.destination, root)
// This package may have been placed on the list to look for when it wasn't
// downloaded but it has since been downloaded before coming to this entry.
if _, err := os.Stat(dest); err == nil {
// Make sure the location contains files. It may be an empty directory.
empty, err := gpath.IsDirectoryEmpty(dest)
if err != nil {
return false, err
}
if empty {
msg.Warn("%s is an existing location with no files. Fetching a new copy of the dependency.", dest)
msg.Debug("Removing empty directory %s", dest)
err := os.RemoveAll(dest)
if err != nil {
msg.Debug("Installer error removing directory %s: %s", dest, err)
return false, err
}
} else {
msg.Debug("Found %s", dest)
return true, nil
}
}
msg.Info("Fetching %s into %s", pkg, m.destination)
d := m.Config.Imports.Get(root)
if d == nil && addTest {
d = m.Config.DevImports.Get(root)
}
// If the dependency is nil it means the Config doesn't yet know about it.
if d == nil {
d, _ = m.Use.Get(root)
// We don't know about this dependency so we create a basic instance.
if d == nil {
d = &cfg.Dependency{Name: root}
}
if addTest {
m.Config.DevImports = append(m.Config.DevImports, d)
} else {
m.Config.Imports = append(m.Config.Imports, d)
}
}
if err := VcsGet(d, dest, m.home, m.cache, m.cacheGopath, m.useGopath); err != nil {
return false, err
}
return true, nil
}
func (m *memCache) put(name, version string) {
m.Lock()
defer m.Unlock()
m.t[name] = true
sv, err := semver.NewVersion(version)
if err != nil {
msg.Debug("Ignoring %s version %s: %s", name, version, err)
return
}
latest, found := m.latest[name]
if found {
lv, err := semver.NewVersion(latest)
if err == nil {
if sv.GreaterThan(lv) {
m.latest[name] = version
}
}
} else {
m.latest[name] = version
}
found = false
for _, v := range m.versions[name] {
if v == version {
found = true
}
}
if !found {
m.versions[name] = append(m.versions[name], version)
}
}
// GodepWorkspace removes any Godeps/_workspace directories and makes sure
// any rewrites are undone.
// Note, this is not concuccency safe.
func GodepWorkspace(v string) error {
vPath = v
if _, err := os.Stat(vPath); err != nil {
if os.IsNotExist(err) {
msg.Debug("Vendor directory does not exist.")
}
return err
}
err := filepath.Walk(vPath, stripGodepWorkspaceHandler)
if err != nil {
return err
}
// Walk the marked projects to make sure rewrites are undone.
for k := range godepMark {
msg.Info("Removing Godep rewrites for %s", k)
err := filepath.Walk(k, rewriteGodepfilesHandler)
if err != nil {
return err
}
}
return nil
}
func stripGodepWorkspaceHandler(path string, info os.FileInfo, err error) error {
// Skip the base vendor directory
if path == vPath {
return nil
}
name := info.Name()
p := filepath.Dir(path)
pn := filepath.Base(p)
if name == "_workspace" && pn == "Godeps" {
if _, err := os.Stat(path); err == nil {
if info.IsDir() {
// Marking this location to make sure rewrites are undone.
pp := filepath.Dir(p)
godepMark[pp] = true
msg.Info("Removing: %s", path)
return os.RemoveAll(path)
}
msg.Debug("%s is not a directory. Skipping removal", path)
return nil
}
}
return nil
}
func (m *MissingPackageHandler) NotFound(pkg string) (bool, error) {
root := util.GetRootFromPackage(pkg)
// Skip any references to the root package.
if root == m.Config.Name {
return false, nil
}
dest := filepath.Join(m.destination, root)
// This package may have been placed on the list to look for when it wasn't
// downloaded but it has since been downloaded before coming to this entry.
if _, err := os.Stat(dest); err == nil {
msg.Debug("Found %s", dest)
return true, nil
}
msg.Info("Fetching %s into %s", pkg, m.destination)
d := m.Config.Imports.Get(root)
// If the dependency is nil it means the Config doesn't yet know about it.
if d == nil {
d = m.Use.Get(root)
// We don't know about this dependency so we create a basic instance.
if d == nil {
d = &cfg.Dependency{Name: root}
}
m.Config.Imports = append(m.Config.Imports, d)
}
if err := VcsGet(d, dest, m.home, m.cache, m.cacheGopath, m.useGopath); err != nil {
return false, err
}
return true, nil
}
// LazyConcurrentUpdate updates only deps that are not already checkout out at the right version.
//
// This is only safe when updating from a lock file.
func LazyConcurrentUpdate(deps []*cfg.Dependency, cwd string, i *Installer, c *cfg.Config) error {
newDeps := []*cfg.Dependency{}
for _, dep := range deps {
destPath := filepath.Join(i.VendorPath(), dep.Name)
// Get a VCS object for this directory
repo, err := dep.GetRepo(destPath)
if err != nil {
newDeps = append(newDeps, dep)
continue
}
ver, err := repo.Version()
if err != nil {
newDeps = append(newDeps, dep)
continue
}
if ver == dep.Reference {
msg.Info("--> Found desired version %s %s!", dep.Name, dep.Reference)
continue
}
msg.Debug("--> Queue %s for update (%s != %s).", dep.Name, ver, dep.Reference)
newDeps = append(newDeps, dep)
}
if len(newDeps) > 0 {
return ConcurrentUpdate(newDeps, cwd, i, c)
}
return nil
}
// Unlock unlocks a particular key name
func Unlock(name string) {
msg.Debug("Unlocking %s", name)
lockSync.Lock()
if m, ok := lockData[name]; ok {
m.Unlock()
}
lockSync.Unlock()
}
func findCurrentBranch(repo vcs.Repo) string {
msg.Debug("Attempting to find current branch for %s", repo.Remote())
// Svn and Bzr don't have default branches.
if repo.Vcs() == vcs.Svn || repo.Vcs() == vcs.Bzr {
return ""
}
if repo.Vcs() == vcs.Git || repo.Vcs() == vcs.Hg {
ver, err := repo.Current()
if err != nil {
msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err)
return ""
}
return ver
}
return ""
}
// StripVcs removes VCS metadata (.git, .hg, .bzr, .svn) from the vendor/
// directory.
func StripVcs() error {
if _, err := os.Stat(VendorDir); err != nil {
if os.IsNotExist(err) {
msg.Debug("Vendor directory does not exist.")
}
return err
}
return filepath.Walk(VendorDir, stripHandler)
}
// EnsureVendorDir ensures that a vendor/ directory is present in the cwd.
func EnsureVendorDir() {
fi, err := os.Stat(gpath.VendorDir)
if err != nil {
msg.Debug("Creating %s", gpath.VendorDir)
if err := os.MkdirAll(gpath.VendorDir, os.ModeDir|0755); err != nil {
msg.Die("Could not create %s: %s", gpath.VendorDir, err)
}
} else if !fi.IsDir() {
msg.Die("Vendor is not a directory")
}
}
// Lock locks a particular key name
func Lock(name string) {
lockSync.Lock()
m, ok := lockData[name]
if !ok {
m = &sync.Mutex{}
lockData[name] = m
}
lockSync.Unlock()
msg.Debug("Locking %s", name)
m.Lock()
}
// sliceToQueue is a special-purpose function for unwrapping a slice of
// dependencies into a queue of fully qualified paths.
func sliceToQueue(deps []*cfg.Dependency, basepath string) *list.List {
l := list.New()
for _, e := range deps {
if len(e.Subpackages) > 0 {
for _, v := range e.Subpackages {
ip := e.Name
if v != "." && v != "" {
ip = ip + "/" + v
}
msg.Debug("Adding local Import %s to queue", ip)
l.PushBack(filepath.Join(basepath, filepath.FromSlash(ip)))
}
} else {
msg.Debug("Adding local Import %s to queue", e.Name)
l.PushBack(filepath.Join(basepath, filepath.FromSlash(e.Name)))
}
}
return l
}
// VcsGet figures out how to fetch a dependency, and then gets it.
//
// VcsGet installs into the cache.
func VcsGet(dep *cfg.Dependency) error {
key, err := cp.Key(dep.Remote())
if err != nil {
msg.Die("Cache key generation error: %s", err)
}
location := cp.Location()
d := filepath.Join(location, "src", key)
repo, err := dep.GetRepo(d)
if err != nil {
return err
}
// If the directory does not exist this is a first cache.
if _, err = os.Stat(d); os.IsNotExist(err) {
msg.Debug("Adding %s to the cache for the first time", dep.Name)
err = repo.Get()
if err != nil {
return err
}
branch := findCurrentBranch(repo)
if branch != "" {
msg.Debug("Saving default branch for %s", repo.Remote())
c := cp.RepoInfo{DefaultBranch: branch}
err = cp.SaveRepoData(key, c)
if err == cp.ErrCacheDisabled {
msg.Debug("Unable to cache default branch because caching is disabled")
} else if err != nil {
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
}
}
} else {
msg.Debug("Updating %s in the cache", dep.Name)
err = repo.Update()
if err != nil {
return err
}
}
return nil
}
// Load pulls the mirrors into memory
func Load() error {
home := gpath.Home()
op := filepath.Join(home, "mirrors.yaml")
var ov *Mirrors
if _, err := os.Stat(op); os.IsNotExist(err) {
msg.Debug("No mirrors.yaml file exists")
ov = &Mirrors{
Repos: make(MirrorRepos, 0),
}
return nil
} else if err != nil {
ov = &Mirrors{
Repos: make(MirrorRepos, 0),
}
return err
}
var err error
ov, err = ReadMirrorsFile(op)
if err != nil {
return fmt.Errorf("Error reading existing mirrors.yaml file: %s", err)
}
msg.Info("Loading mirrors from mirrors.yaml file")
for _, o := range ov.Repos {
msg.Debug("Found mirror: %s to %s (%s)", o.Original, o.Repo, o.Vcs)
no := &mirror{
Repo: o.Repo,
Vcs: o.Vcs,
}
mirrors[o.Original] = no
}
return nil
}
// StripVendor removes nested vendor and Godeps/_workspace/ directories.
func StripVendor() error {
searchPath, _ := Vendor()
if _, err := os.Stat(searchPath); err != nil {
if os.IsNotExist(err) {
msg.Debug("Vendor directory does not exist.")
}
return err
}
err := filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
// Skip the base vendor directory
if path == searchPath {
return nil
}
name := info.Name()
if name == "vendor" {
if _, err := os.Stat(path); err == nil {
if info.IsDir() {
msg.Info("Removing: %s", path)
return os.RemoveAll(path)
}
msg.Debug("%s is not a directory. Skipping removal", path)
return nil
}
}
return nil
})
if err != nil {
return err
}
return strip.GodepWorkspace(searchPath)
}
func stripHandler(path string, info os.FileInfo, err error) error {
name := info.Name()
if name == ".git" || name == ".bzr" || name == ".svn" || name == ".hg" {
if _, err := os.Stat(path); err == nil {
if info.IsDir() {
msg.Info("Removing: %s", path)
return os.RemoveAll(path)
}
msg.Debug("%s is not a directory. Skipping removal", path)
return nil
}
}
return nil
}
// Plugin attempts to find and execute a plugin based on a command.
//
// Exit code 99 means the plugin was never executed. Code 1 means the program
// exited badly.
func Plugin(command string, args []string) {
cwd, err := os.Getwd()
if err != nil {
msg.ExitCode(99)
msg.Die("Could not get working directory: %s", err)
}
cmd := "glide-" + command
var fullcmd string
if fullcmd, err = exec.LookPath(cmd); err != nil {
fullcmd = cwd + "/" + cmd
if _, err := os.Stat(fullcmd); err != nil {
msg.ExitCode(99)
msg.Die("Command %s does not exist.", cmd)
}
}
// Turning os.Args first argument from `glide` to `glide-command`
args[0] = cmd
// Removing the first argument (command)
removed := false
for i, v := range args {
if removed == false && v == command {
args = append(args[:i], args[i+1:]...)
removed = true
}
}
pa := os.ProcAttr{
Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
Dir: cwd,
}
msg.Debug("Delegating to plugin %s (%v)\n", fullcmd, args)
proc, err := os.StartProcess(fullcmd, args, &pa)
if err != nil {
msg.Err("Failed to execute %s: %s", cmd, err)
os.Exit(98)
}
if _, err := proc.Wait(); err != nil {
msg.Err(err.Error())
os.Exit(1)
}
}
// ResolveAll takes a list of packages and returns an inclusive list of all
// vendored dependencies.
//
// While this will scan all of the source code it can find, it will only return
// packages that were either explicitly passed in as deps, or were explicitly
// imported by the code.
//
// Packages that are either CGO or on GOROOT are ignored. Packages that are
// on GOPATH, but not vendored currently generate a warning.
//
// If one of the passed in packages does not exist in the vendor directory,
// an error is returned.
func (r *Resolver) ResolveAll(deps []*cfg.Dependency) ([]string, error) {
queue := sliceToQueue(deps, r.VendorDir)
loc, err := r.ResolveLocal(false)
if err != nil {
return []string{}, err
}
for _, l := range loc {
msg.Debug("Adding local Import %s to queue", l)
queue.PushBack(l)
}
if r.ResolveAllFiles {
return r.resolveList(queue)
}
return r.resolveImports(queue)
}
// StripVendor removes nested vendor and Godeps/_workspace/ directories.
func StripVendor() error {
if _, err := os.Stat(VendorDir); err != nil {
if os.IsNotExist(err) {
msg.Debug("Vendor directory does not exist.")
}
return err
}
err := filepath.Walk(VendorDir, stripVendorHandler)
if err != nil {
return err
}
err = strip.GodepWorkspace(VendorDir)
return err
}
func stripVendorHandler(path string, info os.FileInfo, err error) error {
// Skip the base vendor directory
if path == VendorDir {
return nil
}
name := info.Name()
if name == "vendor" {
if _, err := os.Stat(path); err == nil {
if info.IsDir() {
msg.Info("Removing: %s", path)
return os.RemoveAll(path)
}
msg.Debug("%s is not a directory. Skipping removal", path)
return nil
}
}
return nil
}
// LazyConcurrentUpdate updates only deps that are not already checkout out at the right version.
//
// This is only safe when updating from a lock file.
func LazyConcurrentUpdate(deps []*cfg.Dependency, i *Installer, c *cfg.Config) error {
newDeps := []*cfg.Dependency{}
for _, dep := range deps {
key, err := cache.Key(dep.Remote())
if err != nil {
newDeps = append(newDeps, dep)
continue
}
destPath := filepath.Join(cache.Location(), "src", key)
// Get a VCS object for this directory
repo, err := dep.GetRepo(destPath)
if err != nil {
newDeps = append(newDeps, dep)
continue
}
ver, err := repo.Version()
if err != nil {
newDeps = append(newDeps, dep)
continue
}
if dep.Reference != "" {
ci, err := repo.CommitInfo(dep.Reference)
if err == nil && ci.Commit == dep.Reference {
msg.Info("--> Found desired version locally %s %s!", dep.Name, dep.Reference)
continue
}
}
msg.Debug("--> Queue %s for update (%s != %s).", dep.Name, ver, dep.Reference)
newDeps = append(newDeps, dep)
}
if len(newDeps) > 0 {
return ConcurrentUpdate(newDeps, i, c)
}
return nil
}
// Setup creates the cache location.
func Setup() {
setupMutex.Lock()
defer setupMutex.Unlock()
if isSetup {
return
}
msg.Debug("Setting up the cache directory")
pths := []string{
"cache",
filepath.Join("cache", "src"),
filepath.Join("cache", "info"),
}
for _, l := range pths {
err := os.MkdirAll(filepath.Join(gpath.Home(), l), 0755)
if err != nil {
msg.Die("Cache directory unavailable: %s", err)
}
}
isSetup = true
}
// Setup creates the cache location.
func Setup() error {
setupMutex.Lock()
defer setupMutex.Unlock()
if isSetup {
return nil
}
msg.Debug("Setting up the cache directory")
pths := []string{
"cache",
filepath.Join("cache", "src"),
filepath.Join("cache", "info"),
}
for _, l := range pths {
err := os.MkdirAll(filepath.Join(gpath.Home(), l), 0755)
if err != nil {
return err
}
}
isSetup = true
return nil
}
// VcsVersion set the VCS version for a checkout.
func VcsVersion(dep *cfg.Dependency, vend string) error {
// If the dependency has already been pinned we can skip it. This is a
// faster path so we don't need to resolve it again.
if dep.Pin != "" {
msg.Debug("Dependency %s has already been pinned. Setting version skipped.", dep.Name)
return nil
}
cwd := filepath.Join(vend, dep.Name)
// If there is no refernece configured there is nothing to set.
if dep.Reference == "" {
// Before exiting update the pinned version
repo, err := dep.GetRepo(cwd)
if err != nil {
return err
}
dep.Pin, err = repo.Version()
if err != nil {
return err
}
return nil
}
// When the directory is not empty and has no VCS directory it's
// a vendored files situation.
empty, err := gpath.IsDirectoryEmpty(cwd)
if err != nil {
return err
}
_, err = v.DetectVcsFromFS(cwd)
if empty == false && err == v.ErrCannotDetectVCS {
msg.Warn("%s appears to be a vendored package. Unable to set new version. Consider the '--update-vendored' flag.\n", dep.Name)
} else {
repo, err := dep.GetRepo(cwd)
if err != nil {
return err
}
ver := dep.Reference
// Referenes in Git can begin with a ^ which is similar to semver.
// If there is a ^ prefix we assume it's a semver constraint rather than
// part of the git/VCS commit id.
if repo.IsReference(ver) && !strings.HasPrefix(ver, "^") {
msg.Info("Setting version for %s to %s.\n", dep.Name, ver)
} else {
// Create the constraing first to make sure it's valid before
// working on the repo.
constraint, err := semver.NewConstraint(ver)
// Make sure the constriant is valid. At this point it's not a valid
// reference so if it's not a valid constrint we can exit early.
if err != nil {
msg.Warn("The reference '%s' is not valid\n", ver)
return err
}
// Get the tags and branches (in that order)
refs, err := getAllVcsRefs(repo)
if err != nil {
return err
}
// Convert and filter the list to semver.Version instances
semvers := getSemVers(refs)
// Sort semver list
sort.Sort(sort.Reverse(semver.Collection(semvers)))
found := false
for _, v := range semvers {
if constraint.Check(v) {
found = true
// If the constrint passes get the original reference
ver = v.Original()
break
}
}
if found {
msg.Info("Detected semantic version. Setting version for %s to %s.\n", dep.Name, ver)
} else {
msg.Warn("Unable to find semantic version for constraint %s %s\n", dep.Name, ver)
}
}
if err := repo.UpdateVersion(ver); err != nil {
msg.Error("Failed to set version to %s: %s\n", dep.Reference, err)
return err
}
dep.Pin, err = repo.Version()
if err != nil {
return err
}
}
return nil
}
func wizardFindVersions(d *cfg.Dependency) {
l, err := cache.Location()
if err != nil {
msg.Debug("Problem detecting cache location: %s", err)
return
}
var remote string
if d.Repository != "" {
remote = d.Repository
} else {
remote = "https://" + d.Name
}
key, err := cache.Key(remote)
if err != nil {
msg.Debug("Problem generating cache key for %s: %s", remote, err)
return
}
local := filepath.Join(l, "src", key)
repo, err := vcs.NewRepo(remote, local)
if err != nil {
msg.Debug("Problem getting repo instance: %s", err)
return
}
var useLocal bool
if _, err = os.Stat(local); err == nil {
useLocal = true
}
// Git endpoints allow for querying without fetching the codebase locally.
// We try that first to avoid fetching right away. Is this premature
// optimization?
cc := true
if !useLocal && repo.Vcs() == vcs.Git {
out, err2 := exec.Command("git", "ls-remote", remote).CombinedOutput()
if err2 == nil {
cache.MemTouch(remote)
cc = false
lines := strings.Split(string(out), "\n")
for _, i := range lines {
ti := strings.TrimSpace(i)
if found := createGitParseVersion.FindString(ti); found != "" {
tg := strings.TrimPrefix(strings.TrimSuffix(found, "^{}"), "tags/")
cache.MemPut(remote, tg)
if d.Reference != "" && strings.HasPrefix(ti, d.Reference) {
cache.MemSetCurrent(remote, tg)
}
}
}
}
}
if cc {
cache.Lock(key)
cache.MemTouch(remote)
if _, err = os.Stat(local); os.IsNotExist(err) {
repo.Get()
branch := findCurrentBranch(repo)
c := cache.RepoInfo{DefaultBranch: branch}
err = cache.SaveRepoData(key, c)
if err != nil {
msg.Debug("Error saving cache repo details: %s", err)
}
} else {
repo.Update()
}
tgs, err := repo.Tags()
if err != nil {
msg.Debug("Problem getting tags: %s", err)
} else {
for _, v := range tgs {
cache.MemPut(remote, v)
}
}
if d.Reference != "" && repo.IsReference(d.Reference) {
tgs, err = repo.TagsFromCommit(d.Reference)
if err != nil {
msg.Debug("Problem getting tags for commit: %s", err)
} else {
if len(tgs) > 0 {
for _, v := range tgs {
if !(repo.Vcs() == vcs.Hg && v == "tip") {
cache.MemSetCurrent(remote, v)
}
}
}
}
}
cache.Unlock(key)
}
}
// imports gets all of the imports for a given package.
//
// If the package is in GOROOT, this will return an empty list (but not
// an error).
// If it cannot resolve the pkg, it will return an error.
func (r *Resolver) imports(pkg string) ([]string, error) {
// If this pkg is marked seen, we don't scan it again.
if _, ok := r.seen[pkg]; ok {
msg.Debug("Already saw %s", pkg)
return []string{}, nil
}
// FIXME: On error this should try to NotFound to the dependency, and then import
// it again.
p, err := r.BuildContext.ImportDir(pkg, 0)
if err != nil {
return []string{}, err
}
// It is okay to scan a package more than once. In some cases, this is
// desirable because the package can change between scans (e.g. as a result
// of a failed scan resolving the situation).
msg.Debug("=> Scanning %s (%s)", p.ImportPath, pkg)
r.seen[pkg] = true
// Optimization: If it's in GOROOT, it has no imports worth scanning.
if p.Goroot {
return []string{}, nil
}
// We are only looking for dependencies in vendor. No root, cgo, etc.
buf := []string{}
for _, imp := range p.Imports {
info := r.FindPkg(imp)
switch info.Loc {
case LocUnknown:
// Do we resolve here?
found, err := r.Handler.NotFound(imp)
if err != nil {
msg.Error("Failed to fetch %s: %s", imp, err)
}
if found {
buf = append(buf, filepath.Join(r.VendorDir, filepath.FromSlash(imp)))
continue
}
r.seen[info.Path] = true
case LocVendor:
//msg.Debug("Vendored: %s", imp)
buf = append(buf, info.Path)
case LocGopath:
found, err := r.Handler.OnGopath(imp)
if err != nil {
msg.Error("Failed to fetch %s: %s", imp, err)
}
// If the Handler marks this as found, we drop it into the buffer
// for subsequent processing. Otherwise, we assume that we're
// in a less-than-perfect, but functional, situation.
if found {
buf = append(buf, filepath.Join(r.VendorDir, filepath.FromSlash(imp)))
continue
}
msg.Warn("Package %s is on GOPATH, but not vendored. Ignoring.", imp)
r.seen[info.Path] = true
default:
// Local packages are an odd case. CGO cannot be scanned.
msg.Debug("===> Skipping %s", imp)
}
}
return buf, nil
}
// defaultBranch tries to ascertain the default branch for the given repo.
// Some repos will have multiple branches in them (e.g. Git) while others
// (e.g. Svn) will not.
func defaultBranch(repo v.Repo, home string) string {
// Svn and Bzr use different locations (paths or entire locations)
// for branches so we won't have a default branch.
if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr {
return ""
}
// Check the cache for a value.
key, kerr := cacheCreateKey(repo.Remote())
var d cacheRepoInfo
if kerr == nil {
d, err := cacheRepoData(key, home)
if err == nil {
if d.DefaultBranch != "" {
return d.DefaultBranch
}
}
}
// If we don't have it in the store try some APIs
r := repo.Remote()
u, err := url.Parse(r)
if err != nil {
return ""
}
if u.Scheme == "" {
// Where there is no scheme we try urls like [email protected]:foo/bar
r = strings.Replace(r, ":", "/", -1)
r = "ssh://" + r
u, err = url.Parse(r)
if err != nil {
return ""
}
u.Scheme = ""
}
if u.Host == "github.com" {
parts := strings.Split(u.Path, "/")
if len(parts) != 2 {
return ""
}
api := fmt.Sprintf("https://api.github.com/repos/%s/%s", parts[0], parts[1])
resp, err := http.Get(api)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode >= 300 || resp.StatusCode < 200 {
return ""
}
body, err := ioutil.ReadAll(resp.Body)
var data interface{}
err = json.Unmarshal(body, &data)
if err != nil {
return ""
}
gh := data.(map[string]interface{})
db := gh["default_branch"].(string)
if kerr == nil {
d.DefaultBranch = db
err := saveCacheRepoData(key, d, home)
if err == errCacheDisabled {
msg.Debug("Unable to cache default branch because caching is disabled")
} else if err != nil {
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
}
}
return db
}
if u.Host == "bitbucket.org" {
parts := strings.Split(u.Path, "/")
if len(parts) != 2 {
return ""
}
api := fmt.Sprintf("https://bitbucket.org/api/1.0/repositories/%s/%s/main-branch/", parts[0], parts[1])
resp, err := http.Get(api)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode >= 300 || resp.StatusCode < 200 {
return ""
}
body, err := ioutil.ReadAll(resp.Body)
var data interface{}
err = json.Unmarshal(body, &data)
if err != nil {
return ""
}
bb := data.(map[string]interface{})
db := bb["name"].(string)
if kerr == nil {
d.DefaultBranch = db
err := saveCacheRepoData(key, d, home)
if err == errCacheDisabled {
msg.Debug("Unable to cache default branch because caching is disabled")
} else if err != nil {
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
}
}
return db
}
return ""
}
// VcsGet figures out how to fetch a dependency, and then gets it.
//
// VcsGet installs into the dest.
func VcsGet(dep *cfg.Dependency, dest, home string, cache, cacheGopath, useGopath bool) error {
// When not skipping the $GOPATH look in it for a copy of the package
if useGopath {
// Check if the $GOPATH has a viable version to use and if so copy to vendor
gps := gpath.Gopaths()
for _, p := range gps {
d := filepath.Join(p, "src", dep.Name)
if _, err := os.Stat(d); err == nil {
empty, err := gpath.IsDirectoryEmpty(d)
if empty || err != nil {
continue
}
repo, err := dep.GetRepo(d)
if err != nil {
continue
}
// Dirty repos have uncomitted changes.
if repo.IsDirty() {
continue
}
// Having found a repo we copy it to vendor and update it.
msg.Info("Copying package %s from the GOPATH.", dep.Name)
msg.Debug("Found %s in GOPATH at %s. Copying to %s", dep.Name, d, dest)
err = gpath.CopyDir(d, dest)
if err != nil {
return err
}
// Update the repo in the vendor directory
msg.Debug("Updating %s, now in the vendor path at %s", dep.Name, dest)
repo, err = dep.GetRepo(dest)
if err != nil {
return err
}
err = repo.Update()
if err != nil {
return err
}
// If there is no reference set on the dep we try to checkout
// the default branch.
if dep.Reference == "" {
db := defaultBranch(repo, home)
if db != "" {
err = repo.UpdateVersion(db)
if err != nil && msg.Default.IsDebugging {
msg.Debug("Attempting to set the version on %s to %s failed. Error %s", dep.Name, db, err)
}
}
}
return nil
}
}
}
// When opting in to cache in the GOPATH attempt to do put a copy there.
if cacheGopath {
// Since we didn't find an existing copy in the GOPATHs try to clone there.
gp := gpath.Gopath()
if gp != "" {
d := filepath.Join(gp, "src", dep.Name)
if _, err := os.Stat(d); os.IsNotExist(err) {
// Empty directory so we checkout out the code here.
msg.Debug("Retrieving %s to %s before copying to vendor", dep.Name, d)
repo, err := dep.GetRepo(d)
if err != nil {
return err
}
repo.Get()
branch := findCurrentBranch(repo)
if branch != "" {
// we know the default branch so we can store it in the cache
var loc string
if dep.Repository != "" {
loc = dep.Repository
} else {
loc = "https://" + dep.Name
}
key, err := cacheCreateKey(loc)
if err == nil {
msg.Debug("Saving default branch for %s", repo.Remote())
c := cacheRepoInfo{DefaultBranch: branch}
err = saveCacheRepoData(key, c, home)
if msg.Default.IsDebugging && err == errCacheDisabled {
msg.Debug("Unable to cache default branch because caching is disabled")
}
}
}
msg.Debug("Copying %s from GOPATH at %s to %s", dep.Name, d, dest)
err = gpath.CopyDir(d, dest)
if err != nil {
return err
}
return nil
}
}
}
// If opting in to caching attempt to put it in the cache folder
if cache {
// Check if the cache has a viable version and try to use that.
var loc string
if dep.Repository != "" {
loc = dep.Repository
} else {
loc = "https://" + dep.Name
}
key, err := cacheCreateKey(loc)
if err == nil {
d := filepath.Join(home, "cache", "src", key)
repo, err := dep.GetRepo(d)
if err != nil {
return err
}
// If the directory does not exist this is a first cache.
if _, err = os.Stat(d); os.IsNotExist(err) {
msg.Debug("Adding %s to the cache for the first time", dep.Name)
err = repo.Get()
if err != nil {
return err
}
branch := findCurrentBranch(repo)
if branch != "" {
// we know the default branch so we can store it in the cache
var loc string
if dep.Repository != "" {
loc = dep.Repository
} else {
loc = "https://" + dep.Name
}
key, err := cacheCreateKey(loc)
if err == nil {
msg.Debug("Saving default branch for %s", repo.Remote())
c := cacheRepoInfo{DefaultBranch: branch}
err = saveCacheRepoData(key, c, home)
if err == errCacheDisabled {
msg.Debug("Unable to cache default branch because caching is disabled")
} else if err != nil {
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
}
}
}
} else {
msg.Debug("Updating %s in the cache", dep.Name)
err = repo.Update()
if err != nil {
return err
}
}
msg.Debug("Copying %s from the cache to %s", dep.Name, dest)
err = gpath.CopyDir(d, dest)
if err != nil {
return err
}
return nil
} else {
msg.Warn("Cache key generation error: %s", err)
}
}
// If unable to cache pull directly into the vendor/ directory.
repo, err := dep.GetRepo(dest)
if err != nil {
return err
}
gerr := repo.Get()
// Attempt to cache the default branch
branch := findCurrentBranch(repo)
if branch != "" {
// we know the default branch so we can store it in the cache
var loc string
if dep.Repository != "" {
loc = dep.Repository
} else {
loc = "https://" + dep.Name
}
key, err := cacheCreateKey(loc)
if err == nil {
msg.Debug("Saving default branch for %s", repo.Remote())
c := cacheRepoInfo{DefaultBranch: branch}
err = saveCacheRepoData(key, c, home)
if err == errCacheDisabled {
msg.Debug("Unable to cache default branch because caching is disabled")
} else if err != nil {
msg.Debug("Error saving %s to cache - Error: %s", repo.Remote(), err)
}
}
}
return gerr
}
// VcsUpdate updates to a particular checkout based on the VCS setting.
func VcsUpdate(dep *cfg.Dependency, dest, home string, cache, cacheGopath, useGopath, force, updateVendored bool) error {
// If the dependency has already been pinned we can skip it. This is a
// faster path so we don't need to resolve it again.
if dep.Pin != "" {
msg.Debug("Dependency %s has already been pinned. Fetching updates skipped.", dep.Name)
return nil
}
msg.Info("Fetching updates for %s.\n", dep.Name)
if filterArchOs(dep) {
msg.Info("%s is not used for %s/%s.\n", dep.Name, runtime.GOOS, runtime.GOARCH)
return nil
}
// If destination doesn't exist we need to perform an initial checkout.
if _, err := os.Stat(dest); os.IsNotExist(err) {
if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
msg.Warn("Unable to checkout %s\n", dep.Name)
return err
}
} else {
// At this point we have a directory for the package.
// When the directory is not empty and has no VCS directory it's
// a vendored files situation.
empty, err := gpath.IsDirectoryEmpty(dest)
if err != nil {
return err
}
_, err = v.DetectVcsFromFS(dest)
if updateVendored == false && empty == false && err == v.ErrCannotDetectVCS {
msg.Warn("%s appears to be a vendored package. Unable to update. Consider the '--update-vendored' flag.\n", dep.Name)
} else {
if updateVendored == true && empty == false && err == v.ErrCannotDetectVCS {
// A vendored package, no repo, and updating the vendored packages
// has been opted into.
msg.Info("%s is a vendored package. Updating.", dep.Name)
err = os.RemoveAll(dest)
if err != nil {
msg.Error("Unable to update vendored dependency %s.\n", dep.Name)
return err
} else {
dep.UpdateAsVendored = true
}
if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
msg.Warn("Unable to checkout %s\n", dep.Name)
return err
}
return nil
}
repo, err := dep.GetRepo(dest)
// Tried to checkout a repo to a path that does not work. Either the
// type or endpoint has changed. Force is being passed in so the old
// location can be removed and replaced with the new one.
// Warning, any changes in the old location will be deleted.
// TODO: Put dirty checking in on the existing local checkout.
if (err == v.ErrWrongVCS || err == v.ErrWrongRemote) && force == true {
var newRemote string
if len(dep.Repository) > 0 {
newRemote = dep.Repository
} else {
newRemote = "https://" + dep.Name
}
msg.Warn("Replacing %s with contents from %s\n", dep.Name, newRemote)
rerr := os.RemoveAll(dest)
if rerr != nil {
return rerr
}
if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
msg.Warn("Unable to checkout %s\n", dep.Name)
return err
}
} else if err != nil {
return err
} else if repo.IsDirty() {
return fmt.Errorf("%s contains uncommited changes. Skipping update", dep.Name)
} else {
// Check if the current version is a tag or commit id. If it is
// and that version is already checked out we can skip updating
// which is faster than going out to the Internet to perform
// an update.
if dep.Reference != "" {
version, err := repo.Version()
if err != nil {
return err
}
ib, err := isBranch(dep.Reference, repo)
if err != nil {
return err
}
// If the current version equals the ref and it's not a
// branch it's a tag or commit id so we can skip
// performing an update.
if version == dep.Reference && !ib {
msg.Info("%s is already set to version %s. Skipping update.", dep.Name, dep.Reference)
return nil
}
}
if err := repo.Update(); err != nil {
msg.Warn("Download failed.\n")
return err
}
}
}
}
return nil
}