Example #1
0
func (d *VersionHandler) pkgPath(pkg string) string {
	root, sub := util.NormalizeName(pkg)

	// For the parent applications source skip the cache.
	if root == d.Config.Name {
		pth := gpath.Basepath()
		return filepath.Join(pth, filepath.FromSlash(sub))
	}

	dep := d.Config.Imports.Get(root)
	if dep == nil {
		dep = d.Config.DevImports.Get(root)
	}

	if dep == nil {
		dep, _ = d.Use.Get(root)

		if dep == nil {
			dep = &cfg.Dependency{Name: root}
		}
	}

	key, err := cache.Key(dep.Remote())
	if err != nil {
		msg.Die("Error generating cache key for %s", dep.Name)
	}

	return filepath.Join(cache.Location(), "src", key, filepath.FromSlash(sub))
}
Example #2
0
// SetVersion sets the version for a package. If that package version is already
// set it handles the case by:
// - keeping the already set version
// - proviting messaging about the version conflict
// TODO(mattfarina): The way version setting happens can be improved. Currently not optimal.
func (d *VersionHandler) SetVersion(pkg string) (e error) {
	root := util.GetRootFromPackage(pkg)

	// Skip any references to the root package.
	if root == d.Config.Name {
		return nil
	}

	v := d.Config.Imports.Get(root)

	dep, req := d.Use.Get(root)
	if dep != nil && v != nil {
		if v.Reference == "" && dep.Reference != "" {
			v.Reference = dep.Reference
			// Clear the pin, if set, so the new version can be used.
			v.Pin = ""
			dep = v
		} else if v.Reference != "" && dep.Reference != "" && v.Reference != dep.Reference {
			dest := filepath.Join(d.Destination, filepath.FromSlash(v.Name))
			dep = determineDependency(v, dep, dest, req)
		} else {
			dep = v
		}

	} else if v != nil {
		dep = v
	} else if dep != nil {
		// We've got an imported dependency to use and don't already have a
		// record of it. Append it to the Imports.
		d.Config.Imports = append(d.Config.Imports, dep)
	} else {
		// If we've gotten here we don't have any depenency objects.
		r, sp := util.NormalizeName(pkg)
		dep = &cfg.Dependency{
			Name: r,
		}
		if sp != "" {
			dep.Subpackages = []string{sp}
		}
		d.Config.Imports = append(d.Config.Imports, dep)
	}

	err := VcsVersion(dep, d.Destination)
	if err != nil {
		msg.Warn("Unable to set version on %s to %s. Err: %s", root, dep.Reference, err)
		e = err
	}

	return
}
Example #3
0
// Parse parses a GB-flavored manifest file.
func Parse(dir string) ([]*cfg.Dependency, error) {
	path := filepath.Join(dir, "vendor/manifest")
	if fi, err := os.Stat(path); err != nil || fi.IsDir() {
		return []*cfg.Dependency{}, nil
	}

	msg.Info("Found GB manifest file in %s", gpath.StripBasepath(dir))
	msg.Info("--> Parsing GB metadata...")
	buf := []*cfg.Dependency{}
	file, err := os.Open(path)
	if err != nil {
		return buf, err
	}
	defer file.Close()

	man := Manifest{}

	dec := json.NewDecoder(file)
	if err := dec.Decode(&man); err != nil {
		return buf, err
	}

	seen := map[string]bool{}

	for _, d := range man.Dependencies {
		pkg, sub := util.NormalizeName(d.Importpath)
		if _, ok := seen[pkg]; ok {
			if len(sub) == 0 {
				continue
			}
			for _, dep := range buf {
				if dep.Name == pkg {
					dep.Subpackages = append(dep.Subpackages, sub)
				}
			}
		} else {
			seen[pkg] = true
			dep := &cfg.Dependency{
				Name:       pkg,
				Reference:  d.Revision,
				Repository: d.Repository,
			}
			if len(sub) > 0 {
				dep.Subpackages = []string{sub}
			}
			buf = append(buf, dep)
		}
	}
	return buf, nil
}
Example #4
0
// Parse parses a Godep's Godeps file.
//
// It returns the contents as a dependency array.
func Parse(dir string) ([]*cfg.Dependency, error) {
	path := filepath.Join(dir, "Godeps/Godeps.json")
	if _, err := os.Stat(path); err != nil {
		return []*cfg.Dependency{}, nil
	}
	msg.Info("Found Godeps.json file in %s", gpath.StripBasepath(dir))
	msg.Info("--> Parsing Godeps metadata...")

	buf := []*cfg.Dependency{}

	godeps := &Godeps{}

	// Get a handle to the file.
	file, err := os.Open(path)
	if err != nil {
		return buf, err
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	if err := dec.Decode(godeps); err != nil {
		return buf, err
	}

	seen := map[string]bool{}
	for _, d := range godeps.Deps {
		pkg, sub := util.NormalizeName(d.ImportPath)
		if _, ok := seen[pkg]; ok {
			if len(sub) == 0 {
				continue
			}
			// Modify existing dep with additional subpackages.
			for _, dep := range buf {
				if dep.Name == pkg {
					dep.Subpackages = append(dep.Subpackages, sub)
				}
			}
		} else {
			seen[pkg] = true
			dep := &cfg.Dependency{Name: pkg, Reference: d.Rev}
			if sub != "" {
				dep.Subpackages = []string{sub}
			}
			buf = append(buf, dep)
		}
	}

	return buf, nil
}
Example #5
0
func (d *VersionHandler) pkgPath(pkg string) string {
	root, sub := util.NormalizeName(pkg)

	dep := d.Config.Imports.Get(root)
	if dep == nil {
		dep = d.Config.DevImports.Get(root)
	}

	if dep == nil {
		dep, _ = d.Use.Get(root)

		if dep == nil {
			dep = &cfg.Dependency{Name: root}
		}
	}

	key, err := cache.Key(dep.Remote())
	if err != nil {
		msg.Die("Error generating cache key for %s", dep.Name)
	}

	return filepath.Join(cache.Location(), "src", key, sub)
}
Example #6
0
// PkgPath resolves the location on the filesystem where the package should be.
// This handles making sure to use the cache location.
func (m *MissingPackageHandler) PkgPath(pkg string) string {
	root, sub := util.NormalizeName(pkg)

	d := m.Config.Imports.Get(root)
	if d == nil {
		d = m.Config.DevImports.Get(root)
	}

	if d == nil {
		d, _ = m.Use.Get(root)

		if d == nil {
			d = &cfg.Dependency{Name: root}
		}
	}

	key, err := cache.Key(d.Remote())
	if err != nil {
		msg.Die("Error generating cache key for %s", d.Name)
	}

	return filepath.Join(cache.Location(), "src", key, sub)
}
Example #7
0
// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshaling process
func (d *Dependency) UnmarshalYAML(unmarshal func(interface{}) error) error {
	newDep := &dep{}
	err := unmarshal(&newDep)
	if err != nil {
		return err
	}
	d.Name = newDep.Name
	d.Reference = newDep.Reference
	d.Repository = newDep.Repository
	d.VcsType = newDep.VcsType
	d.Subpackages = newDep.Subpackages
	d.Arch = newDep.Arch
	d.Os = newDep.Os

	if d.Reference == "" && newDep.Ref != "" {
		d.Reference = newDep.Ref
	}

	// Make sure only legitimate VCS are listed.
	d.VcsType = filterVcsType(d.VcsType)

	// Get the root name for the package
	tn, subpkg := util.NormalizeName(d.Name)
	d.Name = tn
	if subpkg != "" {
		d.Subpackages = append(d.Subpackages, subpkg)
	}

	// Older versions of Glide had a / prefix on subpackages in some cases.
	// Here that's cleaned up. Someday we should be able to remove this.
	for k, v := range d.Subpackages {
		d.Subpackages[k] = strings.TrimPrefix(v, "/")
	}

	return nil
}
Example #8
0
// guessDeps attempts to resolve all of the dependencies for a given project.
//
// base is the directory to start with.
// skipImport will skip running the automatic imports.
//
// FIXME: This function is likely a one-off that has a more standard alternative.
// It's also long and could use a refactor.
func guessDeps(base string, skipImport bool) *cfg.Config {
	buildContext, err := util.GetBuildContext()
	if err != nil {
		msg.Die("Failed to build an import context: %s", err)
	}
	name := buildContext.PackageName(base)

	msg.Info("Generating a YAML configuration file and guessing the dependencies")

	config := new(cfg.Config)

	// Get the name of the top level package
	config.Name = name

	// Import by looking at other package managers and looking over the
	// entire directory structure.

	// Attempt to import from other package managers.
	if !skipImport {
		guessImportDeps(base, config)
	}

	importLen := len(config.Imports)
	if importLen == 0 {
		msg.Info("Scanning code to look for dependencies")
	} else {
		msg.Info("Scanning code to look for dependencies not found in import")
	}

	// Resolve dependencies by looking at the tree.
	r, err := dependency.NewResolver(base)
	if err != nil {
		msg.Die("Error creating a dependency resolver: %s", err)
	}

	h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
	r.Handler = h

	sortable, err := r.ResolveLocal(false)
	if err != nil {
		msg.Die("Error resolving local dependencies: %s", err)
	}

	sort.Strings(sortable)

	vpath := r.VendorDir
	if !strings.HasSuffix(vpath, "/") {
		vpath = vpath + string(os.PathSeparator)
	}

	for _, pa := range sortable {
		n := strings.TrimPrefix(pa, vpath)
		root, subpkg := util.NormalizeName(n)

		if !config.HasDependency(root) && root != config.Name {
			msg.Info("--> Found reference to %s\n", n)
			d := &cfg.Dependency{
				Name: root,
			}
			if len(subpkg) > 0 {
				d.Subpackages = []string{subpkg}
			}
			config.Imports = append(config.Imports, d)
		} else if config.HasDependency(root) {
			if len(subpkg) > 0 {
				subpkg = strings.TrimPrefix(subpkg, "/")
				d := config.Imports.Get(root)
				if !d.HasSubpackage(subpkg) {
					msg.Info("--> Adding sub-package %s to %s\n", subpkg, root)
					d.Subpackages = append(d.Subpackages, subpkg)
				}
			}
		}
	}

	if len(config.Imports) == importLen && importLen != 0 {
		msg.Info("--> Code scanning found no additional imports")
	}

	return config
}
Example #9
0
// addPkgsToConfig adds the given packages to the config file.
//
// Along the way it:
// - ensures that this package is not in the ignore list
// - checks to see if this is already in the dependency list.
// - splits version of of package name and adds the version attribute
// - separates repo from packages
// - sets up insecure repo URLs where necessary
// - generates a list of subpackages
func addPkgsToConfig(conf *cfg.Config, names []string, insecure, nonInteract bool) (int, error) {

	if len(names) == 1 {
		msg.Info("Preparing to install %d package.", len(names))
	} else {
		msg.Info("Preparing to install %d packages.", len(names))
	}
	numAdded := 0
	for _, name := range names {
		var version string
		parts := strings.Split(name, "#")
		if len(parts) > 1 {
			name = parts[0]
			version = parts[1]
		}

		msg.Info("Attempting to get package %s", name)

		root, subpkg := util.NormalizeName(name)
		if len(root) == 0 {
			return 0, fmt.Errorf("Package name is required for %q.", name)
		}

		if conf.HasDependency(root) {

			// Check if the subpackage is present.
			if subpkg != "" {
				dep := conf.Imports.Get(root)
				if dep.HasSubpackage(subpkg) {
					msg.Warn("--> Package %q is already in glide.yaml. Skipping", name)
				} else {
					dep.Subpackages = append(dep.Subpackages, subpkg)
					msg.Info("--> Adding sub-package %s to existing import %s", subpkg, root)
					numAdded++
				}
			} else {
				msg.Warn("--> Package %q is already in glide.yaml. Skipping", root)
			}
			continue
		}

		if conf.HasIgnore(root) {
			msg.Warn("--> Package %q is set to be ignored in glide.yaml. Skipping", root)
			continue
		}

		dep := &cfg.Dependency{
			Name: root,
		}

		// When retriving from an insecure location set the repo to the
		// insecure location.
		if insecure {
			dep.Repository = "http://" + root
		}

		if version != "" {
			dep.Reference = version
		} else if !nonInteract {
			getWizard(dep)
		}

		if len(subpkg) > 0 {
			dep.Subpackages = []string{subpkg}
		}

		if dep.Reference != "" {
			msg.Info("--> Adding %s to your configuration with the version %s", dep.Name, dep.Reference)
		} else {
			msg.Info("--> Adding %s to your configuration %s", dep.Name)
		}

		conf.Imports = append(conf.Imports, dep)
		numAdded++
	}
	return numAdded, nil
}
Example #10
0
// SetVersion sets the version for a package. If that package version is already
// set it handles the case by:
// - keeping the already set version
// - proviting messaging about the version conflict
// TODO(mattfarina): The way version setting happens can be improved. Currently not optimal.
func (d *VersionHandler) SetVersion(pkg string, addTest bool) (e error) {
	root := util.GetRootFromPackage(pkg)

	// Skip any references to the root package.
	if root == d.Config.Name {
		return nil
	}

	v := d.Config.Imports.Get(root)
	if addTest {
		if v == nil {
			v = d.Config.DevImports.Get(root)
		} else if d.Config.DevImports.Has(root) {
			// Both imports and test imports lists the same dependency.
			// There are import chains (because the import tree is resolved
			// before the test tree) that can cause this.
			tempD := d.Config.DevImports.Get(root)
			if tempD.Reference != v.Reference {
				msg.Warn("Using import %s (version %s) for test instead of testImport (version %s).", v.Name, v.Reference, tempD.Reference)
			}
			// TODO(mattfarina): Note repo difference in a warning.
		}
	}

	dep, req := d.Use.Get(root)
	if dep != nil && v != nil {
		if v.Reference == "" && dep.Reference != "" {
			v.Reference = dep.Reference
			// Clear the pin, if set, so the new version can be used.
			v.Pin = ""
			dep = v
		} else if v.Reference != "" && dep.Reference != "" && v.Reference != dep.Reference {
			dest := d.pkgPath(pkg)
			dep = determineDependency(v, dep, dest, req)
		} else {
			dep = v
		}

	} else if v != nil {
		dep = v
	} else if dep != nil {
		// We've got an imported dependency to use and don't already have a
		// record of it. Append it to the Imports.
		if addTest {
			d.Config.DevImports = append(d.Config.DevImports, dep)
		} else {
			d.Config.Imports = append(d.Config.Imports, dep)
		}
	} else {
		// If we've gotten here we don't have any depenency objects.
		r, sp := util.NormalizeName(pkg)
		dep = &cfg.Dependency{
			Name: r,
		}
		if sp != "" {
			dep.Subpackages = []string{sp}
		}
		if addTest {
			d.Config.DevImports = append(d.Config.DevImports, dep)
		} else {
			d.Config.Imports = append(d.Config.Imports, dep)
		}
	}

	err := VcsVersion(dep)
	if err != nil {
		msg.Warn("Unable to set version on %s to %s. Err: %s", root, dep.Reference, err)
		e = err
	}

	return
}
Example #11
0
// Update updates all dependencies.
//
// It begins with the dependencies in the config file, but also resolves
// transitive dependencies. The returned lockfile has all of the dependencies
// listed, but the version reconciliation has not been done.
//
// In other words, all versions in the Lockfile will be empty.
func (i *Installer) Update(conf *cfg.Config) error {
	base := "."

	ic := newImportCache()

	m := &MissingPackageHandler{
		home:    i.Home,
		force:   i.Force,
		Config:  conf,
		Use:     ic,
		updated: i.Updated,
	}

	v := &VersionHandler{
		Use:       ic,
		Imported:  make(map[string]bool),
		Conflicts: make(map[string]bool),
		Config:    conf,
	}

	// Update imports
	res, err := dependency.NewResolver(base)
	res.ResolveTest = i.ResolveTest
	if err != nil {
		msg.Die("Failed to create a resolver: %s", err)
	}
	res.Config = conf
	res.Handler = m
	res.VersionHandler = v
	res.ResolveAllFiles = i.ResolveAllFiles
	msg.Info("Resolving imports")

	imps, timps, err := res.ResolveLocal(false)
	if err != nil {
		msg.Die("Failed to resolve local packages: %s", err)
	}
	var deps cfg.Dependencies
	var tdeps cfg.Dependencies
	for _, v := range imps {
		n := res.Stripv(v)
		if conf.HasIgnore(n) {
			continue
		}
		rt, sub := util.NormalizeName(n)
		if sub == "" {
			sub = "."
		}
		d := deps.Get(rt)
		if d == nil {
			nd := &cfg.Dependency{
				Name:        rt,
				Subpackages: []string{sub},
			}
			deps = append(deps, nd)
		} else if !d.HasSubpackage(sub) {
			d.Subpackages = append(d.Subpackages, sub)
		}
	}
	if i.ResolveTest {
		for _, v := range timps {
			n := res.Stripv(v)
			if conf.HasIgnore(n) {
				continue
			}
			rt, sub := util.NormalizeName(n)
			if sub == "" {
				sub = "."
			}
			d := deps.Get(rt)
			if d == nil {
				d = tdeps.Get(rt)
			}
			if d == nil {
				nd := &cfg.Dependency{
					Name:        rt,
					Subpackages: []string{sub},
				}
				tdeps = append(tdeps, nd)
			} else if !d.HasSubpackage(sub) {
				d.Subpackages = append(d.Subpackages, sub)
			}
		}
	}

	_, err = allPackages(deps, res, false)
	if err != nil {
		msg.Die("Failed to retrieve a list of dependencies: %s", err)
	}

	if i.ResolveTest {
		msg.Debug("Resolving test dependencies")
		_, err = allPackages(tdeps, res, true)
		if err != nil {
			msg.Die("Failed to retrieve a list of test dependencies: %s", err)
		}
	}

	msg.Info("Downloading dependencies. Please wait...")

	err = ConcurrentUpdate(conf.Imports, i, conf)
	if err != nil {
		return err
	}

	if i.ResolveTest {
		err = ConcurrentUpdate(conf.DevImports, i, conf)
		if err != nil {
			return err
		}
	}

	return nil
}
Example #12
0
// resolveList takes a list and resolves it.
//
// This walks the entire file tree for the given dependencies, not just the
// parts that are imported directly. Using this will discover dependencies
// regardless of OS, and arch.
func (r *Resolver) resolveList(queue *list.List) ([]string, error) {

	var failedDep string
	for e := queue.Front(); e != nil; e = e.Next() {
		dep := e.Value.(string)
		t := strings.TrimPrefix(dep, r.VendorDir+string(os.PathSeparator))
		if r.Config.HasIgnore(t) {
			msg.Info("Ignoring: %s", t)
			continue
		}
		r.VersionHandler.Process(t)
		//msg.Warn("#### %s ####", dep)
		//msg.Info("Seen Count: %d", len(r.seen))
		// Catch the outtermost dependency.
		failedDep = dep
		err := filepath.Walk(dep, func(path string, fi os.FileInfo, err error) error {
			if err != nil && err != filepath.SkipDir {
				return err
			}

			// Skip files.
			if !fi.IsDir() {
				return nil
			}
			// Skip dirs that are not source.
			if !srcDir(fi) {
				//msg.Debug("Skip resource %s", fi.Name())
				return filepath.SkipDir
			}

			// Anything that comes through here has already been through
			// the queue.
			r.alreadyQ[path] = true
			e := r.queueUnseen(path, queue)
			if err != nil {
				failedDep = path
				//msg.Error("Failed to fetch dependency %s: %s", path, err)
			}
			return e
		})
		if err != nil && err != filepath.SkipDir {
			msg.Error("Dependency %s failed to resolve: %s.", failedDep, err)
			return []string{}, err
		}
	}

	res := make([]string, 0, queue.Len())

	// In addition to generating a list
	for e := queue.Front(); e != nil; e = e.Next() {
		t := strings.TrimPrefix(e.Value.(string), r.VendorDir+string(os.PathSeparator))
		root, sp := util.NormalizeName(t)

		// TODO(mattfarina): Need to eventually support devImport
		existing := r.Config.Imports.Get(root)
		if existing != nil {
			if sp != "" && !existing.HasSubpackage(sp) {
				existing.Subpackages = append(existing.Subpackages, sp)
			}
		} else {
			newDep := &cfg.Dependency{
				Name: root,
			}
			if sp != "" {
				newDep.Subpackages = []string{sp}
			}

			r.Config.Imports = append(r.Config.Imports, newDep)
		}
		res = append(res, e.Value.(string))
	}

	return res, nil
}
Example #13
0
// resolveImports takes a list of existing packages and resolves their imports.
//
// It returns a list of all of the packages that it can determine are required
// for the given code to function.
//
// The expectation is that each item in the queue is an absolute path to a
// vendored package. This attempts to read that package, and then find
// its referenced packages. Those packages are then added to the list
// to be scanned next.
//
// The resolver's handler is used in the cases where a package cannot be
// located.
func (r *Resolver) resolveImports(queue *list.List) ([]string, error) {
	for e := queue.Front(); e != nil; e = e.Next() {
		vdep := e.Value.(string)
		dep := r.stripv(vdep)

		// Check if marked in the Q and then explicitly mark it. We want to know
		// if it had previously been marked and ensure it for the future.
		_, foundQ := r.alreadyQ[dep]
		r.alreadyQ[dep] = true

		// If we've already encountered an error processing this dependency
		// skip it.
		_, foundErr := r.hadError[dep]
		if foundErr {
			continue
		}

		// Skip ignored packages
		if r.Config.HasIgnore(dep) {
			msg.Info("Ignoring: %s", dep)
			continue
		}
		r.VersionHandler.Process(dep)

		// Here, we want to import the package and see what imports it has.
		msg.Debug("Trying to open %s", vdep)
		pkg, err := r.BuildContext.ImportDir(vdep, 0)
		if err != nil {
			msg.Debug("ImportDir error on %s: %s", vdep, err)
			if strings.HasPrefix(err.Error(), "no buildable Go source") {
				msg.Debug("No subpackages declared. Skipping %s.", dep)
				continue
			} else if os.IsNotExist(err) && !foundErr && !foundQ {
				// If the location doesn't exist, there hasn't already been an
				// error, it's not already been in the Q then try to fetch it.
				// When there's an error or it's already in the Q (it should be
				// fetched if it's marked in r.alreadyQ) we skip to make sure
				// not to get stuck in a recursion.

				// If the location doesn't exist try to fetch it.
				if ok, err2 := r.Handler.NotFound(dep); ok {
					r.alreadyQ[dep] = true

					// By adding to the queue it will get reprocessed now that
					// it exists.
					queue.PushBack(r.vpath(dep))
					r.VersionHandler.SetVersion(dep)
				} else if err2 != nil {
					r.hadError[dep] = true
					msg.Error("Error looking for %s: %s", dep, err2)
				} else {
					r.hadError[dep] = true
					// TODO (mpb): Should we toss this into a Handler to
					// see if this is on GOPATH and copy it?
					msg.Info("Not found in vendor/: %s (1)", dep)
				}
			} else {
				r.hadError[dep] = true
				msg.Error("Error scanning %s: %s", dep, err)
			}
			continue
		}

		// Range over all of the identified imports and see which ones we
		// can locate.
		for _, imp := range pkg.Imports {
			pi := r.FindPkg(imp)
			if pi.Loc != LocCgo && pi.Loc != LocGoroot {
				msg.Debug("Package %s imports %s", dep, imp)
			}
			switch pi.Loc {
			case LocVendor:
				msg.Debug("In vendor: %s", imp)
				if _, ok := r.alreadyQ[imp]; !ok {
					msg.Debug("Marking %s to be scanned.", imp)
					r.alreadyQ[imp] = true
					queue.PushBack(r.vpath(imp))
					if err := r.Handler.InVendor(imp); err == nil {
						r.VersionHandler.SetVersion(imp)
					} else {
						msg.Warn("Error updating %s: %s", imp, err)
					}
					r.VersionHandler.SetVersion(imp)
				}
			case LocUnknown:
				msg.Debug("Missing %s. Trying to resolve.", imp)
				if ok, err := r.Handler.NotFound(imp); ok {
					r.alreadyQ[imp] = true
					queue.PushBack(r.vpath(imp))
					r.VersionHandler.SetVersion(imp)
				} else if err != nil {
					r.hadError[dep] = true
					msg.Warn("Error looking for %s: %s", imp, err)
				} else {
					r.hadError[dep] = true
					msg.Info("Not found: %s (2)", imp)
				}
			case LocGopath:
				msg.Debug("Found on GOPATH, not vendor: %s", imp)
				if _, ok := r.alreadyQ[imp]; !ok {
					// Only scan it if it gets moved into vendor/
					if ok, _ := r.Handler.OnGopath(imp); ok {
						r.alreadyQ[imp] = true
						queue.PushBack(r.vpath(imp))
						r.VersionHandler.SetVersion(imp)
					}
				}
			}
		}

	}

	// FIXME: From here to the end is a straight copy of the resolveList() func.
	res := make([]string, 0, queue.Len())

	// In addition to generating a list
	for e := queue.Front(); e != nil; e = e.Next() {
		t := r.stripv(e.Value.(string))
		root, sp := util.NormalizeName(t)

		// TODO(mattfarina): Need to eventually support devImport
		existing := r.Config.Imports.Get(root)
		if existing != nil {
			if sp != "" && !existing.HasSubpackage(sp) {
				existing.Subpackages = append(existing.Subpackages, sp)
			}
		} else {
			newDep := &cfg.Dependency{
				Name: root,
			}
			if sp != "" {
				newDep.Subpackages = []string{sp}
			}

			r.Config.Imports = append(r.Config.Imports, newDep)
		}
		res = append(res, t)
	}

	return res, nil
}
Example #14
0
// guessDeps attempts to resolve all of the dependencies for a given project.
//
// base is the directory to start with.
// skipImport will skip running the automatic imports.
//
// FIXME: This function is likely a one-off that has a more standard alternative.
// It's also long and could use a refactor.
func guessDeps(base string, skipImport bool) *cfg.Config {
	buildContext, err := util.GetBuildContext()
	if err != nil {
		msg.Die("Failed to build an import context: %s", err)
	}
	name := buildContext.PackageName(base)

	msg.Info("Generating a YAML configuration file and guessing the dependencies")

	config := new(cfg.Config)

	// Get the name of the top level package
	config.Name = name

	// Import by looking at other package managers and looking over the
	// entire directory structure.

	// Attempt to import from other package managers.
	if !skipImport {
		msg.Info("Attempting to import from other package managers (use --skip-import to skip)")
		deps := []*cfg.Dependency{}
		absBase, err := filepath.Abs(base)
		if err != nil {
			msg.Die("Failed to resolve location of %s: %s", base, err)
		}

		if d, ok := guessImportGodep(absBase); ok {
			msg.Info("Importing Godep configuration")
			msg.Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide")
			deps = d
		} else if d, ok := guessImportGPM(absBase); ok {
			msg.Info("Importing GPM configuration")
			deps = d
		} else if d, ok := guessImportGB(absBase); ok {
			msg.Info("Importing GB configuration")
			deps = d
		}

		for _, i := range deps {
			msg.Info("Found imported reference to %s\n", i.Name)
			config.Imports = append(config.Imports, i)
		}
	}

	// Resolve dependencies by looking at the tree.
	r, err := dependency.NewResolver(base)
	if err != nil {
		msg.Die("Error creating a dependency resolver: %s", err)
	}

	h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
	r.Handler = h

	sortable, err := r.ResolveLocal(false)
	if err != nil {
		msg.Die("Error resolving local dependencies: %s", err)
	}

	sort.Strings(sortable)

	vpath := r.VendorDir
	if !strings.HasSuffix(vpath, "/") {
		vpath = vpath + string(os.PathSeparator)
	}

	for _, pa := range sortable {
		n := strings.TrimPrefix(pa, vpath)
		root, subpkg := util.NormalizeName(n)

		if !config.HasDependency(root) {
			msg.Info("Found reference to %s\n", n)
			d := &cfg.Dependency{
				Name: root,
			}
			if len(subpkg) > 0 {
				d.Subpackages = []string{subpkg}
			}
			config.Imports = append(config.Imports, d)
		} else {
			if len(subpkg) > 0 {
				subpkg = strings.TrimPrefix(subpkg, "/")
				d := config.Imports.Get(root)
				if !d.HasSubpackage(subpkg) {
					msg.Info("Adding sub-package %s to %s\n", subpkg, root)
					d.Subpackages = append(d.Subpackages, subpkg)
				}
			}
		}
	}

	return config
}
Example #15
0
// resolveImports takes a list of existing packages and resolves their imports.
//
// It returns a list of all of the packages that it can determine are required
// for the given code to function.
//
// The expectation is that each item in the queue is an absolute path to a
// vendored package. This attempts to read that package, and then find
// its referenced packages. Those packages are then added to the list
// to be scanned next.
//
// The resolver's handler is used in the cases where a package cannot be
// located.
//
// testDeps specifies if the test dependencies should be resolved and addTest
// specifies if the dependencies should be added to the Config.DevImports. This
// is important because we may resolve normal dependencies of test deps and add
// them to the DevImports list.
func (r *Resolver) resolveImports(queue *list.List, testDeps, addTest bool) ([]string, error) {
	msg.Debug("Resolving import path")

	// When test deps passed in but not resolving return empty.
	if (testDeps || addTest) && !r.ResolveTest {
		return []string{}, nil
	}

	for e := queue.Front(); e != nil; e = e.Next() {
		vdep := e.Value.(string)
		dep := r.Stripv(vdep)
		// Check if marked in the Q and then explicitly mark it. We want to know
		// if it had previously been marked and ensure it for the future.

		_, foundQ := r.alreadyQ[dep]
		r.alreadyQ[dep] = true

		// If we've already encountered an error processing this dependency
		// skip it.
		_, foundErr := r.hadError[dep]
		if foundErr {
			continue
		}

		// Skip ignored packages
		if r.Config.HasIgnore(dep) {
			msg.Debug("Ignoring: %s", dep)
			continue
		}
		r.VersionHandler.Process(dep)
		// Here, we want to import the package and see what imports it has.
		msg.Debug("Trying to open %s", vdep)
		var imps []string
		pkg, err := r.BuildContext.ImportDir(r.Handler.PkgPath(dep), 0)
		if err != nil && strings.HasPrefix(err.Error(), "found packages ") {
			// If we got here it's because a package and multiple packages
			// declared. This is often because of an example with a package
			// or main but +build ignore as a build tag. In that case we
			// try to brute force the packages with a slower scan.
			msg.Debug("Using Iterative Scanning for %s", dep)
			if testDeps {
				_, imps, err = IterativeScan(r.Handler.PkgPath(dep))
			} else {
				imps, _, err = IterativeScan(r.Handler.PkgPath(dep))
			}

			if err != nil {
				msg.Err("Iterative scanning error %s: %s", dep, err)
				continue
			}
		} else if err != nil {
			errStr := err.Error()
			msg.Debug("ImportDir error on %s: %s", r.Handler.PkgPath(dep), err)
			if strings.HasPrefix(errStr, "no buildable Go source") {
				msg.Debug("No subpackages declared. Skipping %s.", dep)
				continue
			} else if os.IsNotExist(err) && !foundErr && !foundQ {
				// If the location doesn't exist, there hasn't already been an
				// error, it's not already been in the Q then try to fetch it.
				// When there's an error or it's already in the Q (it should be
				// fetched if it's marked in r.alreadyQ) we skip to make sure
				// not to get stuck in a recursion.

				// If the location doesn't exist try to fetch it.
				if ok, err2 := r.Handler.NotFound(dep, addTest); ok {
					r.alreadyQ[dep] = true

					// By adding to the queue it will get reprocessed now that
					// it exists.
					queue.PushBack(r.vpath(dep))
					r.VersionHandler.SetVersion(dep, addTest)
				} else if err2 != nil {
					r.hadError[dep] = true
					msg.Err("Error looking for %s: %s", dep, err2)
				} else {
					r.hadError[dep] = true
					// TODO (mpb): Should we toss this into a Handler to
					// see if this is on GOPATH and copy it?
					msg.Info("Not found in vendor/: %s (1)", dep)
				}
			} else if strings.Contains(errStr, "no such file or directory") {
				r.hadError[dep] = true
				msg.Err("Error scanning %s: %s", dep, err)
				msg.Err("This error means the referenced package was not found.")
				msg.Err("Missing file or directory errors usually occur when multiple packages")
				msg.Err("share a common dependency and the first reference encountered by the scanner")
				msg.Err("sets the version to one that does not contain a subpackage needed required")
				msg.Err("by another package that uses the shared dependency. Try setting a")
				msg.Err("version in your glide.yaml that works for all packages that share this")
				msg.Err("dependency.")
			} else {
				r.hadError[dep] = true
				msg.Err("Error scanning %s: %s", dep, err)
			}
			continue
		} else {
			if testDeps {
				imps = dedupeStrings(pkg.TestImports, pkg.XTestImports)
			} else {
				imps = pkg.Imports
			}

		}

		// Range over all of the identified imports and see which ones we
		// can locate.
		for _, imp := range imps {
			if r.Config.HasIgnore(imp) {
				msg.Debug("Ignoring: %s", imp)
				continue
			}
			pi := r.FindPkg(imp)
			if pi.Loc != LocCgo && pi.Loc != LocGoroot && pi.Loc != LocAppengine {
				msg.Debug("Package %s imports %s", dep, imp)
			}
			switch pi.Loc {
			case LocVendor:
				msg.Debug("In vendor: %s", imp)
				if _, ok := r.alreadyQ[imp]; !ok {
					msg.Debug("Marking %s to be scanned.", imp)
					r.alreadyQ[dep] = true
					queue.PushBack(r.vpath(imp))
					if err := r.Handler.InVendor(imp, addTest); err == nil {
						r.VersionHandler.SetVersion(imp, addTest)
					} else {
						msg.Warn("Error updating %s: %s", imp, err)
					}
				}
			case LocUnknown:
				msg.Debug("Missing %s. Trying to resolve.", imp)
				if ok, err := r.Handler.NotFound(imp, addTest); ok {
					r.alreadyQ[dep] = true
					queue.PushBack(r.vpath(imp))
					r.VersionHandler.SetVersion(imp, addTest)
				} else if err != nil {
					r.hadError[dep] = true
					msg.Err("Error looking for %s: %s", imp, err)
				} else {
					r.hadError[dep] = true
					msg.Err("Not found: %s (2)", imp)
				}
			case LocGopath:
				msg.Debug("Found on GOPATH, not vendor: %s", imp)
				if _, ok := r.alreadyQ[imp]; !ok {
					// Only scan it if it gets moved into vendor/
					if ok, _ := r.Handler.OnGopath(imp, addTest); ok {
						r.alreadyQ[dep] = true
						queue.PushBack(r.vpath(imp))
						r.VersionHandler.SetVersion(imp, addTest)
					}
				}
			}
		}

	}

	if len(r.hadError) > 0 {
		// Errors occurred so we return.
		return []string{}, errors.New("Error resolving imports")
	}

	// FIXME: From here to the end is a straight copy of the resolveList() func.
	res := make([]string, 0, queue.Len())

	// In addition to generating a list
	for e := queue.Front(); e != nil; e = e.Next() {
		t := r.Stripv(e.Value.(string))
		root, sp := util.NormalizeName(t)

		// Skip ignored packages
		if r.Config.HasIgnore(e.Value.(string)) {
			msg.Debug("Ignoring: %s", e.Value.(string))
			continue
		}

		// TODO(mattfarina): Need to eventually support devImport
		existing := r.Config.Imports.Get(root)
		if existing == nil && addTest {
			existing = r.Config.DevImports.Get(root)
		}
		if existing != nil {
			if sp != "" && !existing.HasSubpackage(sp) {
				existing.Subpackages = append(existing.Subpackages, sp)
			}
		} else {
			newDep := &cfg.Dependency{
				Name: root,
			}
			if sp != "" {
				newDep.Subpackages = []string{sp}
			}

			if addTest {
				r.Config.DevImports = append(r.Config.DevImports, newDep)
			} else {
				r.Config.Imports = append(r.Config.Imports, newDep)
			}
		}
		res = append(res, t)
	}

	return res, nil
}
Example #16
0
// Parse parses a Gomfile.
func Parse(dir string) ([]*cfg.Dependency, error) {
	path := filepath.Join(dir, "Gomfile")
	if fi, err := os.Stat(path); err != nil || fi.IsDir() {
		return []*cfg.Dependency{}, nil
	}

	msg.Info("Found Gomfile in %s", gpath.StripBasepath(dir))
	msg.Info("--> Parsing Gomfile metadata...")
	buf := []*cfg.Dependency{}

	goms, err := parseGomfile(path)
	if err != nil {
		return []*cfg.Dependency{}, err
	}

	for _, gom := range goms {
		// Do we need to skip this dependency?
		if val, ok := gom.options["skipdep"]; ok && val.(string) == "true" {
			continue
		}

		// Check for custom cloning command
		if _, ok := gom.options["command"]; ok {
			return []*cfg.Dependency{}, errors.New("Glide does not support custom Gomfile commands")
		}

		// Check for groups/environments
		if val, ok := gom.options["group"]; ok {
			groups := toStringSlice(val)
			if !stringsContain(groups, "development") && !stringsContain(groups, "production") {
				// right now we only support development and production
				msg.Info("Skipping dependency '%s' because it isn't in the development or production group", gom.name)
				continue
			}
		}

		pkg, sub := util.NormalizeName(gom.name)

		dep := &cfg.Dependency{
			Name: pkg,
		}

		if len(sub) > 0 {
			dep.Subpackages = []string{sub}
		}

		// Check for a specific revision
		if val, ok := gom.options["commit"]; ok {
			dep.Reference = val.(string)
		}
		if val, ok := gom.options["tag"]; ok {
			dep.Reference = val.(string)
		}
		if val, ok := gom.options["branch"]; ok {
			dep.Reference = val.(string)
		}

		// Parse goos and goarch
		if val, ok := gom.options["goos"]; ok {
			dep.Os = toStringSlice(val)
		}
		if val, ok := gom.options["goarch"]; ok {
			dep.Arch = toStringSlice(val)
		}

		buf = append(buf, dep)
	}

	return buf, nil
}