// NormalizeName takes a package name and normalizes it to the top level package.
//
// For example, golang.org/x/crypto/ssh becomes golang.org/x/crypto. 'ssh' is
// returned as extra data.
func NormalizeName(name string) (string, string) {
// Fastpath check if a name in the GOROOT. There is an issue when a pkg
// is in the GOROOT and GetRootFromPackage tries to look it up because it
// expects remote names.
b, err := util.GetBuildContext()
if err == nil {
p := filepath.Join(b.GOROOT, "src", filepath.FromSlash(name))
if _, err := os.Stat(p); err == nil {
return name, ""
}
}
root := util.GetRootFromPackage(name)
extra := strings.TrimPrefix(name, root)
if len(extra) > 0 && extra != "/" {
extra = strings.TrimPrefix(extra, "/")
} else {
// If extra is / (which is what it would be here) we want to return ""
extra = ""
}
return root, extra
// parts := strings.SplitN(name, "/", 4)
// extra := ""
// if len(parts) < 3 {
// return name, extra
// }
// if len(parts) == 4 {
// extra = parts[3]
// }
// return strings.Join(parts[0:3], "/"), extra
}
// NewResolver returns a new Resolver initialized with the DefaultMissingPackageHandler.
//
// This will return an error if the given path does not meet the basic criteria
// for a Go source project. For example, basedir must have a vendor subdirectory.
//
// The BuildContext uses the "go/build".Default to resolve dependencies.
func NewResolver(basedir string) (*Resolver, error) {
var err error
basedir, err = filepath.Abs(basedir)
if err != nil {
return nil, err
}
vdir := filepath.Join(basedir, "vendor")
buildContext, err := util.GetBuildContext()
if err != nil {
return nil, err
}
r := &Resolver{
Handler: &DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}},
basedir: basedir,
VendorDir: vdir,
BuildContext: buildContext,
seen: map[string]bool{},
alreadyQ: map[string]bool{},
findCache: map[string]*PkgInfo{},
}
// TODO: Make sure the build context is correctly set up. Especially in
// regards to GOROOT, which is not always set.
return r, nil
}
// EnsureConfig loads and returns a config file.
//
// Any error will cause an immediate exit, with an error printed to Stderr.
func EnsureConfig() *cfg.Config {
yamlpath, err := gpath.Glide()
if err != nil {
msg.ExitCode(2)
msg.Die("Failed to find %s file in directory tree: %s", gpath.GlideFile, err)
}
yml, err := ioutil.ReadFile(yamlpath)
if err != nil {
msg.ExitCode(2)
msg.Die("Failed to load %s: %s", yamlpath, err)
}
conf, err := cfg.ConfigFromYaml(yml)
if err != nil {
msg.ExitCode(3)
msg.Die("Failed to parse %s: %s", yamlpath, err)
}
b := filepath.Dir(yamlpath)
buildContext, err := util.GetBuildContext()
if err != nil {
msg.Die("Failed to build an import context while ensuring config: %s", err)
}
cwd, err := os.Getwd()
if err != nil {
msg.Err("Unable to get the current working directory")
} else {
// Determining a package name requires a relative path
b, err = filepath.Rel(b, cwd)
if err == nil {
name := buildContext.PackageName(b)
if name != conf.Name {
msg.Warn("The name listed in the config file (%s) does not match the current location (%s)", conf.Name, name)
}
} else {
msg.Warn("Problem finding the config file path (%s) relative to the current directory (%s): %s", b, cwd, err)
}
}
err = mirrors.Load()
if err != nil {
msg.Err("Unable to load mirrors: %s", err)
}
return conf
}
// Tree prints a tree representing dependencies.
func Tree(basedir string, showcore bool) {
buildContext, err := util.GetBuildContext()
if err != nil {
msg.Die("Failed to get a build context: %s", err)
}
myName := buildContext.PackageName(basedir)
if basedir == "." {
var err error
basedir, err = os.Getwd()
if err != nil {
msg.Die("Could not get working directory")
}
}
msg.Puts(myName)
l := list.New()
l.PushBack(myName)
tree.Display(buildContext, basedir, myName, 1, showcore, l)
}
// NewResolver returns a new Resolver initialized with the DefaultMissingPackageHandler.
//
// This will return an error if the given path does not meet the basic criteria
// for a Go source project. For example, basedir must have a vendor subdirectory.
//
// The BuildContext uses the "go/build".Default to resolve dependencies.
func NewResolver(basedir string) (*Resolver, error) {
var err error
basedir, err = filepath.Abs(basedir)
if err != nil {
return nil, err
}
basedir, err = checkForBasedirSymlink(basedir)
if err != nil {
return nil, err
}
vdir := filepath.Join(basedir, "vendor")
buildContext, err := util.GetBuildContext()
if err != nil {
return nil, err
}
r := &Resolver{
Handler: &DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}},
VersionHandler: &DefaultVersionHandler{},
basedir: basedir,
VendorDir: vdir,
BuildContext: buildContext,
seen: map[string]bool{},
alreadyQ: map[string]bool{},
hadError: map[string]bool{},
findCache: map[string]*PkgInfo{},
// The config instance here should really be replaced with a real one.
Config: &cfg.Config{},
}
// TODO: Make sure the build context is correctly set up. Especially in
// regards to GOROOT, which is not always set.
return r, nil
}
// Tree prints a tree representing dependencies.
func Tree(basedir string, showcore bool) {
msg.Warn("The tree command is deprecated and will be removed in a future version")
buildContext, err := util.GetBuildContext()
if err != nil {
msg.Die("Failed to get a build context: %s", err)
}
myName := buildContext.PackageName(basedir)
if basedir == "." {
var err error
basedir, err = os.Getwd()
if err != nil {
msg.Die("Could not get working directory")
}
}
msg.Puts(myName)
l := list.New()
l.PushBack(myName)
tree.Display(buildContext, basedir, myName, 1, showcore, l)
}
// Tree prints a tree representing dependencies.
func Tree(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
buildContext, err := util.GetBuildContext()
if err != nil {
return nil, err
}
showcore := p.Get("showcore", false).(bool)
basedir := p.Get("dir", ".").(string)
myName := guessPackageName(buildContext, basedir)
if basedir == "." {
var err error
basedir, err = os.Getwd()
if err != nil {
Error("Could not get working directory")
return nil, err
}
}
fmt.Println(myName)
l := list.New()
l.PushBack(myName)
displayTree(buildContext, basedir, myName, 1, showcore, l)
return nil, nil
}
// guessDeps attempts to resolve all of the dependencies for a given project.
//
// base is the directory to start with.
// skipImport will skip running the automatic imports.
//
// FIXME: This function is likely a one-off that has a more standard alternative.
// It's also long and could use a refactor.
func guessDeps(base string, skipImport bool) *cfg.Config {
buildContext, err := util.GetBuildContext()
if err != nil {
msg.Die("Failed to build an import context: %s", err)
}
name := buildContext.PackageName(base)
msg.Info("Generating a YAML configuration file and guessing the dependencies")
config := new(cfg.Config)
// Get the name of the top level package
config.Name = name
// Import by looking at other package managers and looking over the
// entire directory structure.
// Attempt to import from other package managers.
if !skipImport {
guessImportDeps(base, config)
}
importLen := len(config.Imports)
if importLen == 0 {
msg.Info("Scanning code to look for dependencies")
} else {
msg.Info("Scanning code to look for dependencies not found in import")
}
// Resolve dependencies by looking at the tree.
r, err := dependency.NewResolver(base)
if err != nil {
msg.Die("Error creating a dependency resolver: %s", err)
}
h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
r.Handler = h
sortable, err := r.ResolveLocal(false)
if err != nil {
msg.Die("Error resolving local dependencies: %s", err)
}
sort.Strings(sortable)
vpath := r.VendorDir
if !strings.HasSuffix(vpath, "/") {
vpath = vpath + string(os.PathSeparator)
}
for _, pa := range sortable {
n := strings.TrimPrefix(pa, vpath)
root, subpkg := util.NormalizeName(n)
if !config.HasDependency(root) && root != config.Name {
msg.Info("--> Found reference to %s\n", n)
d := &cfg.Dependency{
Name: root,
}
if len(subpkg) > 0 {
d.Subpackages = []string{subpkg}
}
config.Imports = append(config.Imports, d)
} else if config.HasDependency(root) {
if len(subpkg) > 0 {
subpkg = strings.TrimPrefix(subpkg, "/")
d := config.Imports.Get(root)
if !d.HasSubpackage(subpkg) {
msg.Info("--> Adding sub-package %s to %s\n", subpkg, root)
d.Subpackages = append(d.Subpackages, subpkg)
}
}
}
}
if len(config.Imports) == importLen && importLen != 0 {
msg.Info("--> Code scanning found no additional imports")
}
return config
}
// IterativeScan attempts to obtain a list of imported dependencies from a
// package. This scanning is different from ImportDir as part of the go/build
// package. It looks over different permutations of the supported OS/Arch to
// try and find all imports. This is different from setting UseAllFiles to
// true on the build Context. It scopes down to just the supported OS/Arch.
//
// Note, there are cases where multiple packages are in the same directory. This
// usually happens with an example that has a main package and a +build tag
// of ignore. This is a bit of a hack. It causes UseAllFiles to have errors.
func IterativeScan(path string) ([]string, error) {
// TODO(mattfarina): Add support for release tags.
tgs, _ := readBuildTags(path)
// Handle the case of scanning with no tags
tgs = append(tgs, "")
var pkgs []string
for _, tt := range tgs {
// split the tag combination to look at permutations.
ts := strings.Split(tt, ",")
var ttgs []string
var arch string
var ops string
for _, ttt := range ts {
dirty := false
if strings.HasPrefix(ttt, "!") {
dirty = true
ttt = strings.TrimPrefix(ttt, "!")
}
if isSupportedOs(ttt) {
if dirty {
ops = getOsValue(ttt)
} else {
ops = ttt
}
} else if isSupportedArch(ttt) {
if dirty {
arch = getArchValue(ttt)
} else {
arch = ttt
}
} else {
if !dirty {
ttgs = append(ttgs, ttt)
}
}
}
// Handle the case where there are no tags but we need to iterate
// on something.
if len(ttgs) == 0 {
ttgs = append(ttgs, "")
}
b, err := util.GetBuildContext()
if err != nil {
return []string{}, err
}
// Make sure use all files is off
b.UseAllFiles = false
// Set the OS and Arch for this pass
b.GOARCH = arch
b.GOOS = ops
b.BuildTags = ttgs
msg.Debug("Scanning with Arch(%s), OS(%s), and Build Tags(%v)", arch, ops, ttgs)
pk, err := b.ImportDir(path, 0)
// If there are no buildable souce with this permutation we skip it.
if err != nil && strings.HasPrefix(err.Error(), "no buildable Go source files in") {
continue
} else if err != nil && strings.HasPrefix(err.Error(), "found packages ") {
// A permutation may cause multiple packages to appear. For example,
// an example file with an ignore build tag. If this happens we
// ignore it.
// TODO(mattfarina): Find a better way.
msg.Debug("Found multiple packages while scanning %s: %s", path, err)
continue
} else if err != nil {
msg.Debug("Problem parsing package at %s for %s %s", path, ops, arch)
return []string{}, err
}
for _, dep := range pk.Imports {
found := false
for _, p := range pkgs {
if p == dep {
found = true
}
}
if !found {
pkgs = append(pkgs, dep)
}
}
}
return pkgs, nil
}
// guessDeps attempts to resolve all of the dependencies for a given project.
//
// base is the directory to start with.
// skipImport will skip running the automatic imports.
//
// FIXME: This function is likely a one-off that has a more standard alternative.
// It's also long and could use a refactor.
func guessDeps(base string, skipImport bool) *cfg.Config {
buildContext, err := util.GetBuildContext()
if err != nil {
msg.Die("Failed to build an import context: %s", err)
}
name := buildContext.PackageName(base)
msg.Info("Generating a YAML configuration file and guessing the dependencies")
config := new(cfg.Config)
// Get the name of the top level package
config.Name = name
// Import by looking at other package managers and looking over the
// entire directory structure.
// Attempt to import from other package managers.
if !skipImport {
msg.Info("Attempting to import from other package managers (use --skip-import to skip)")
deps := []*cfg.Dependency{}
absBase, err := filepath.Abs(base)
if err != nil {
msg.Die("Failed to resolve location of %s: %s", base, err)
}
if d, ok := guessImportGodep(absBase); ok {
msg.Info("Importing Godep configuration")
msg.Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide")
deps = d
} else if d, ok := guessImportGPM(absBase); ok {
msg.Info("Importing GPM configuration")
deps = d
} else if d, ok := guessImportGB(absBase); ok {
msg.Info("Importing GB configuration")
deps = d
}
for _, i := range deps {
msg.Info("Found imported reference to %s\n", i.Name)
config.Imports = append(config.Imports, i)
}
}
// Resolve dependencies by looking at the tree.
r, err := dependency.NewResolver(base)
if err != nil {
msg.Die("Error creating a dependency resolver: %s", err)
}
h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
r.Handler = h
sortable, err := r.ResolveLocal(false)
if err != nil {
msg.Die("Error resolving local dependencies: %s", err)
}
sort.Strings(sortable)
vpath := r.VendorDir
if !strings.HasSuffix(vpath, "/") {
vpath = vpath + string(os.PathSeparator)
}
for _, pa := range sortable {
n := strings.TrimPrefix(pa, vpath)
root, subpkg := util.NormalizeName(n)
if !config.HasDependency(root) {
msg.Info("Found reference to %s\n", n)
d := &cfg.Dependency{
Name: root,
}
if len(subpkg) > 0 {
d.Subpackages = []string{subpkg}
}
config.Imports = append(config.Imports, d)
} else {
if len(subpkg) > 0 {
subpkg = strings.TrimPrefix(subpkg, "/")
d := config.Imports.Get(root)
if !d.HasSubpackage(subpkg) {
msg.Info("Adding sub-package %s to %s\n", subpkg, root)
d.Subpackages = append(d.Subpackages, subpkg)
}
}
}
}
return config
}
// GuessDeps tries to get the dependencies for the current directory.
//
// Params
// - dirname (string): Directory to use as the base. Default: "."
// - skipImport (book): Whether to skip importing from Godep, GPM, and gb
func GuessDeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
buildContext, err := util.GetBuildContext()
if err != nil {
return nil, err
}
base := p.Get("dirname", ".").(string)
skipImport := p.Get("skipImport", false).(bool)
name := guessPackageName(buildContext, base)
Info("Generating a YAML configuration file and guessing the dependencies")
config := new(cfg.Config)
// Get the name of the top level package
config.Name = name
// Import by looking at other package managers and looking over the
// entire directory structure.
// Attempt to import from other package managers.
if !skipImport {
Info("Attempting to import from other package managers (use --skip-import to skip)")
deps := []*cfg.Dependency{}
absBase, err := filepath.Abs(base)
if err != nil {
return nil, err
}
if d, ok := guessImportGodep(absBase); ok {
Info("Importing Godep configuration")
Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide")
deps = d
} else if d, ok := guessImportGPM(absBase); ok {
Info("Importing GPM configuration")
deps = d
} else if d, ok := guessImportGB(absBase); ok {
Info("Importing GB configuration")
deps = d
}
for _, i := range deps {
Info("Found imported reference to %s\n", i.Name)
config.Imports = append(config.Imports, i)
}
}
// Resolve dependencies by looking at the tree.
r, err := dependency.NewResolver(base)
if err != nil {
return nil, err
}
h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
r.Handler = h
sortable, err := r.ResolveLocal(false)
if err != nil {
return nil, err
}
sort.Strings(sortable)
vpath := r.VendorDir
if !strings.HasSuffix(vpath, "/") {
vpath = vpath + string(os.PathSeparator)
}
for _, pa := range sortable {
n := strings.TrimPrefix(pa, vpath)
root := util.GetRootFromPackage(n)
if !config.HasDependency(root) {
Info("Found reference to %s\n", n)
d := &cfg.Dependency{
Name: root,
}
subpkg := strings.TrimPrefix(n, root)
if len(subpkg) > 0 && subpkg != "/" {
d.Subpackages = []string{subpkg}
}
config.Imports = append(config.Imports, d)
} else {
subpkg := strings.TrimPrefix(n, root)
if len(subpkg) > 0 && subpkg != "/" {
subpkg = strings.TrimPrefix(subpkg, "/")
d := config.Imports.Get(root)
f := false
for _, v := range d.Subpackages {
if v == subpkg {
f = true
}
}
if !f {
Info("Adding sub-package %s to %s\n", subpkg, root)
d.Subpackages = append(d.Subpackages, subpkg)
}
}
}
}
return config, nil
}
// mergeGuess guesses dependencies and merges.
//
// This always returns true because it always handles the job of searching
// for dependencies. So generally it should be the last merge strategy
// that you try.
func mergeGuess(dir, pkg string, f *flattening, scanned map[string]bool) ([]string, bool) {
deps := f.deps
Info("Scanning %s for dependencies.", pkg)
buildContext, err := util.GetBuildContext()
if err != nil {
Warn("Could not scan package %q: %s", pkg, err)
return []string{}, false
}
res := []string{}
if _, err := os.Stat(dir); err != nil {
Warn("Directory is missing: %s", dir)
return res, true
}
d := walkDeps(buildContext, dir, pkg)
for _, oname := range d {
if _, ok := scanned[oname]; ok {
//Info("===> Scanned %s already. Skipping", name)
continue
}
Debug("=> Scanning %s", oname)
name, _ := NormalizeName(oname)
//if _, ok := deps[name]; ok {
//scanned[oname] = true
//Debug("====> Seen %s already. Skipping", name)
//continue
//}
if f.conf.HasIgnore(name) {
Debug("==> Skipping %s because it is on the ignore list", name)
continue
}
found := findPkg(buildContext, name, dir)
switch found.PType {
case ptypeUnknown:
Info("==> Unknown %s (%s)", name, oname)
Debug("✨☆ Undownloaded dependency: %s", name)
repo := util.GetRootFromPackage(name)
nd := &cfg.Dependency{
Name: name,
Repository: "https://" + repo,
}
deps[name] = nd
res = append(res, name)
case ptypeGoroot, ptypeCgo:
scanned[oname] = true
// Why do we break rather than continue?
break
default:
// We're looking for dependencies that might exist in $GOPATH
// but not be on vendor. We add any that are on $GOPATH.
if _, ok := deps[name]; !ok {
Debug("✨☆ GOPATH dependency: %s", name)
nd := &cfg.Dependency{Name: name}
deps[name] = nd
res = append(res, name)
}
scanned[oname] = true
}
}
return res, true
}