// FromYaml takes a yaml string and converts it to a Config instance. func FromYaml(yml string) (*Config, error) { c := &Config{} err := yaml.Unmarshal([]byte(yml), &c) if err != nil { return nil, err } // The ref property is for the legacy yaml file structure. // This sets the currect version to the ref if a version isn't // already set. for _, v := range c.Imports { if v.Reference == "" && v.Ref != "" { v.Reference = v.Ref } v.Ref = "" // Make sure only legitimate VCS are listed. v.VcsType = filterVcsType(v.VcsType) // Get the root name for the package o := v.Name v.Name = util.GetRootFromPackage(v.Name) subpkg := strings.TrimPrefix(o, v.Name) if len(subpkg) > 0 && subpkg != o { v.Subpackages = append(v.Subpackages, strings.TrimPrefix(subpkg, "/")) } } for _, v := range c.DevImports { if v.Reference == "" && v.Ref != "" { v.Reference = v.Ref } v.Ref = "" v.VcsType = filterVcsType(v.VcsType) // Get the root name for the package o := v.Name v.Name = util.GetRootFromPackage(v.Name) subpkg := strings.TrimPrefix(o, v.Name) if len(subpkg) > 0 && subpkg != o { v.Subpackages = append(v.Subpackages, subpkg) } } c.Imports, err = c.Imports.DeDupe() if err != nil { return c, err } c.DevImports, err = c.DevImports.DeDupe() if err != nil { return c, err } return c, nil }
func (m *MissingPackageHandler) NotFound(pkg string) (bool, error) { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == m.RootPackage { return false, nil } dest := filepath.Join(m.destination, root) // This package may have been placed on the list to look for when it wasn't // downloaded but it has since been downloaded before coming to this entry. if _, err := os.Stat(dest); err == nil { return true, nil } msg.Info("Fetching %s into %s", pkg, m.destination) d := m.Config.Imports.Get(root) // If the dependency is nil it means the Config doesn't yet know about it. if d == nil { d = m.Use.Get(root) // We don't know about this dependency so we create a basic instance. if d == nil { d = &cfg.Dependency{Name: root} } m.Config.Imports = append(m.Config.Imports, d) } if err := VcsGet(d, dest, m.home, m.cache, m.cacheGopath, m.useGopath); err != nil { return false, err } return true, nil }
// InVendor updates a package in the vendor/ directory to make sure the latest // is available. func (m *MissingPackageHandler) InVendor(pkg string) error { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == m.Config.Name { return nil } dest := filepath.Join(m.destination, root) d := m.Config.Imports.Get(root) // If the dependency is nil it means the Config doesn't yet know about it. if d == nil { d, _ = m.Use.Get(root) // We don't know about this dependency so we create a basic instance. if d == nil { d = &cfg.Dependency{Name: root} } m.Config.Imports = append(m.Config.Imports, d) } if err := VcsUpdate(d, dest, m.home, m.cache, m.cacheGopath, m.useGopath, m.force, m.updateVendored, m.updated); err != nil { return err } return nil }
// GetAll gets zero or more repos. // // This takes a package name, normalizes it, finds the repo, and installs it. // It's the workhorse behind `glide get`. // // Params: // - packages ([]string): Package names to get. // - verbose (bool): default false // // Returns: // - []*Dependency: A list of constructed dependencies. func GetAll(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { names := p.Get("packages", []string{}).([]string) cfg := p.Get("conf", nil).(*yaml.Config) insecure := p.Get("insecure", false).(bool) home := p.Get("home", "").(string) cache := p.Get("cache", false).(bool) cacheGopath := p.Get("cacheGopath", false).(bool) skipGopath := p.Get("skipGopath", false).(bool) Info("Preparing to install %d package.", len(names)) deps := []*yaml.Dependency{} for _, name := range names { cwd, err := VendorPath(c) if err != nil { return nil, err } root := util.GetRootFromPackage(name) if len(root) == 0 { return nil, fmt.Errorf("Package name is required for %q.", name) } if cfg.HasDependency(root) { Warn("Package %q is already in glide.yaml. Skipping", root) continue } dest := path.Join(cwd, root) if err != nil { Error("Could not construct repo for %q: %s", name, err) return false, err } dep := &yaml.Dependency{ Name: root, } // When retriving from an insecure location set the repo to the // insecure location. if insecure { dep.Repository = "http://" + root } subpkg := strings.TrimPrefix(name, root) if len(subpkg) > 0 && subpkg != "/" { dep.Subpackages = []string{subpkg} } if err := VcsGet(dep, dest, home, cache, cacheGopath, skipGopath); err != nil { return dep, err } cfg.Imports = append(cfg.Imports, dep) deps = append(deps, dep) } return deps, nil }
func (m *MissingPackageHandler) fetchToCache(pkg string, addTest bool) error { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == m.Config.Name { return nil } d := m.Config.Imports.Get(root) if d == nil && addTest { d = m.Config.DevImports.Get(root) } // If the dependency is nil it means the Config doesn't yet know about it. if d == nil { d, _ = m.Use.Get(root) // We don't know about this dependency so we create a basic instance. if d == nil { d = &cfg.Dependency{Name: root} } if addTest { m.Config.DevImports = append(m.Config.DevImports, d) } else { m.Config.Imports = append(m.Config.Imports, d) } } return VcsUpdate(d, m.force, m.updated) }
// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshaling process func (d *Dependency) UnmarshalYAML(unmarshal func(interface{}) error) error { newDep := &dep{} err := unmarshal(&newDep) if err != nil { return err } d.Name = newDep.Name d.Reference = newDep.Reference d.Repository = newDep.Repository d.VcsType = newDep.VcsType d.Subpackages = newDep.Subpackages d.Arch = newDep.Arch d.Os = newDep.Os if d.Reference == "" && newDep.Ref != "" { d.Reference = newDep.Ref } // Make sure only legitimate VCS are listed. d.VcsType = filterVcsType(d.VcsType) // Get the root name for the package o := d.Name d.Name = util.GetRootFromPackage(d.Name) subpkg := strings.TrimPrefix(o, d.Name) if len(subpkg) > 0 && subpkg != "/" { d.Subpackages = append(d.Subpackages, strings.TrimPrefix(subpkg, "/")) } return nil }
// NormalizeName takes a package name and normalizes it to the top level package. // // For example, golang.org/x/crypto/ssh becomes golang.org/x/crypto. 'ssh' is // returned as extra data. func NormalizeName(name string) (string, string) { // Fastpath check if a name in the GOROOT. There is an issue when a pkg // is in the GOROOT and GetRootFromPackage tries to look it up because it // expects remote names. b, err := util.GetBuildContext() if err == nil { p := filepath.Join(b.GOROOT, "src", filepath.FromSlash(name)) if _, err := os.Stat(p); err == nil { return name, "" } } root := util.GetRootFromPackage(name) extra := strings.TrimPrefix(name, root) if len(extra) > 0 && extra != "/" { extra = strings.TrimPrefix(extra, "/") } else { // If extra is / (which is what it would be here) we want to return "" extra = "" } return root, extra // parts := strings.SplitN(name, "/", 4) // extra := "" // if len(parts) < 3 { // return name, extra // } // if len(parts) == 4 { // extra = parts[3] // } // return strings.Join(parts[0:3], "/"), extra }
func (m *MissingPackageHandler) OnGopath(pkg string) (bool, error) { // If useGopath is false, we fall back to the strategy of fetching from // remote. if !m.useGopath { return m.NotFound(pkg) } root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == m.RootPackage { return false, nil } msg.Info("Copying package %s from the GOPATH.", pkg) dest := filepath.Join(m.destination, pkg) // Find package on Gopath for _, gp := range gpath.Gopaths() { src := filepath.Join(gp, pkg) // FIXME: Should probably check if src is a dir or symlink. if _, err := os.Stat(src); err == nil { if err := os.MkdirAll(dest, os.ModeDir|0755); err != nil { return false, err } if err := gpath.CopyDir(src, dest); err != nil { return false, err } return true, nil } } msg.Error("Could not locate %s on the GOPATH, though it was found before.", pkg) return false, nil }
// Process imports dependencies for a package func (d *VersionHandler) Process(pkg string) (e error) { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == d.RootPackage { return nil } // We have not tried to import, yet. // Should we look in places other than the root of the project? if d.Imported[root] == false { d.Imported[root] = true p := filepath.Join(d.Destination, root) f, deps, err := importer.Import(p) if f && err == nil { for _, dep := range deps { // The fist one wins. Would something smater than this be better? exists := d.Use.Get(dep.Name) if exists == nil && (dep.Reference != "" || dep.Repository != "") { d.Use.Add(dep.Name, dep) } } } else if err != nil { msg.Error("Unable to import from %s. Err: %s", root, err) e = err } } return }
// NotFound attempts to retrieve a package when not found in the local vendor/ // folder. It will attempt to get it from the remote location info. func (m *MissingPackageHandler) NotFound(pkg string, addTest bool) (bool, error) { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == m.Config.Name { return false, nil } dest := filepath.Join(m.destination, root) // This package may have been placed on the list to look for when it wasn't // downloaded but it has since been downloaded before coming to this entry. if _, err := os.Stat(dest); err == nil { // Make sure the location contains files. It may be an empty directory. empty, err := gpath.IsDirectoryEmpty(dest) if err != nil { return false, err } if empty { msg.Warn("%s is an existing location with no files. Fetching a new copy of the dependency.", dest) msg.Debug("Removing empty directory %s", dest) err := os.RemoveAll(dest) if err != nil { msg.Debug("Installer error removing directory %s: %s", dest, err) return false, err } } else { msg.Debug("Found %s", dest) return true, nil } } msg.Info("Fetching %s into %s", pkg, m.destination) d := m.Config.Imports.Get(root) if d == nil && addTest { d = m.Config.DevImports.Get(root) } // If the dependency is nil it means the Config doesn't yet know about it. if d == nil { d, _ = m.Use.Get(root) // We don't know about this dependency so we create a basic instance. if d == nil { d = &cfg.Dependency{Name: root} } if addTest { m.Config.DevImports = append(m.Config.DevImports, d) } else { m.Config.Imports = append(m.Config.Imports, d) } } if err := VcsGet(d, dest, m.home, m.cache, m.cacheGopath, m.useGopath); err != nil { return false, err } return true, nil }
func parseGodepGodeps(dir string) ([]*cfg.Dependency, error) { path := filepath.Join(dir, "Godeps", "Godeps.json") if _, err := os.Stat(path); err != nil { return []*cfg.Dependency{}, nil } Info("Found Godeps.json file.\n") buf := []*cfg.Dependency{} godeps := new(Godeps) // Get a handle to the file. file, err := os.Open(path) if err != nil { return buf, err } defer file.Close() dec := json.NewDecoder(file) if err := dec.Decode(godeps); err != nil { return buf, err } // Info("Importing %d packages from %s.\n", len(godeps.Deps), godeps.ImportPath) seen := map[string]bool{} for _, d := range godeps.Deps { // Info("Adding package %s\n", d.ImportPath) pkg := util.GetRootFromPackage(d.ImportPath) sub := strings.TrimPrefix(d.ImportPath, pkg) sub = strings.TrimPrefix(sub, "/") if _, ok := seen[pkg]; ok { if len(sub) == 0 { continue } // Modify existing dep with additional subpackages. for _, dep := range buf { if dep.Name == pkg { dep.Subpackages = append(dep.Subpackages, sub) } } } else { seen[pkg] = true dep := &cfg.Dependency{Name: pkg, Reference: d.Rev} if len(sub) > 0 { dep.Subpackages = []string{sub} } buf = append(buf, dep) } } return buf, nil }
// SetVersion sets the version for a package. If that package version is already // set it handles the case by: // - keeping the already set version // - proviting messaging about the version conflict // TODO(mattfarina): The way version setting happens can be improved. Currently not optimal. func (d *VersionHandler) SetVersion(pkg string) (e error) { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == d.Config.Name { return nil } v := d.Config.Imports.Get(root) dep, req := d.Use.Get(root) if dep != nil && v != nil { if v.Reference == "" && dep.Reference != "" { v.Reference = dep.Reference // Clear the pin, if set, so the new version can be used. v.Pin = "" dep = v } else if v.Reference != "" && dep.Reference != "" && v.Reference != dep.Reference { dest := filepath.Join(d.Destination, filepath.FromSlash(v.Name)) dep = determineDependency(v, dep, dest, req) } else { dep = v } } else if v != nil { dep = v } else if dep != nil { // We've got an imported dependency to use and don't already have a // record of it. Append it to the Imports. d.Config.Imports = append(d.Config.Imports, dep) } else { // If we've gotten here we don't have any depenency objects. r, sp := util.NormalizeName(pkg) dep = &cfg.Dependency{ Name: r, } if sp != "" { dep.Subpackages = []string{sp} } d.Config.Imports = append(d.Config.Imports, dep) } err := VcsVersion(dep, d.Destination) if err != nil { msg.Warn("Unable to set version on %s to %s. Err: %s", root, dep.Reference, err) e = err } return }
// SetVersion sets the version for a package. If that package version is already // set it handles the case by: // - keeping the already set version // - proviting messaging about the version conflict // TODO(mattfarina): The way version setting happens can be improved. Currently not optimal. func (d *VersionHandler) SetVersion(pkg string, addTest bool) (e error) { root := util.GetRootFromPackage(pkg) // Skip any references to the root package. if root == d.Config.Name { return nil } v := d.Config.Imports.Get(root) if addTest { if v == nil { v = d.Config.DevImports.Get(root) } else if d.Config.DevImports.Has(root) { // Both imports and test imports lists the same dependency. // There are import chains (because the import tree is resolved // before the test tree) that can cause this. tempD := d.Config.DevImports.Get(root) if tempD.Reference != v.Reference { msg.Warn("Using import %s (version %s) for test instead of testImport (version %s).", v.Name, v.Reference, tempD.Reference) } // TODO(mattfarina): Note repo difference in a warning. } } dep, req := d.Use.Get(root) if dep != nil && v != nil { if v.Reference == "" && dep.Reference != "" { v.Reference = dep.Reference // Clear the pin, if set, so the new version can be used. v.Pin = "" dep = v } else if v.Reference != "" && dep.Reference != "" && v.Reference != dep.Reference { dest := d.pkgPath(pkg) dep = determineDependency(v, dep, dest, req) } else { dep = v } } else if v != nil { dep = v } else if dep != nil { // We've got an imported dependency to use and don't already have a // record of it. Append it to the Imports. if addTest { d.Config.DevImports = append(d.Config.DevImports, dep) } else { d.Config.Imports = append(d.Config.Imports, dep) } } else { // If we've gotten here we don't have any depenency objects. r, sp := util.NormalizeName(pkg) dep = &cfg.Dependency{ Name: r, } if sp != "" { dep.Subpackages = []string{sp} } if addTest { d.Config.DevImports = append(d.Config.DevImports, dep) } else { d.Config.Imports = append(d.Config.Imports, dep) } } err := VcsVersion(dep) if err != nil { msg.Warn("Unable to set version on %s to %s. Err: %s", root, dep.Reference, err) e = err } return }
// GetAll gets zero or more repos. // // This takes a package name, normalizes it, finds the repo, and installs it. // It's the workhorse behind `glide get`. // // Params: // - packages ([]string): Package names to get. // - verbose (bool): default false // // Returns: // - []*Dependency: A list of constructed dependencies. func GetAll(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { names := p.Get("packages", []string{}).([]string) conf := p.Get("conf", nil).(*cfg.Config) insecure := p.Get("insecure", false).(bool) Info("Preparing to install %d package.", len(names)) deps := []*cfg.Dependency{} for _, name := range names { var version string parts := strings.Split(name, "#") if len(parts) > 1 { name = parts[0] version = parts[1] } root := util.GetRootFromPackage(name) if len(root) == 0 { return nil, fmt.Errorf("Package name is required for %q.", name) } if conf.HasDependency(root) { Warn("Package %q is already in glide.yaml. Skipping", root) continue } if conf.HasIgnore(root) { Warn("Package %q is set to be ignored in glide.yaml. Skipping", root) continue } dep := &cfg.Dependency{ Name: root, } if version != "" { dep.Reference = version } // When retriving from an insecure location set the repo to the // insecure location. if insecure { dep.Repository = "http://" + root } subpkg := strings.TrimPrefix(name, root) if len(subpkg) > 0 && subpkg != "/" { dep.Subpackages = []string{subpkg} } if dep.Reference != "" { Info("Importing %s with the version %s", dep.Name, dep.Reference) } else { Info("Importing %s", dep.Name) } conf.Imports = append(conf.Imports, dep) deps = append(deps, dep) } Info("Walking the dependency tree to calculate versions") return deps, nil }
// GuessDeps tries to get the dependencies for the current directory. // // Params // - dirname (string): Directory to use as the base. Default: "." // - skipImport (book): Whether to skip importing from Godep, GPM, and gb func GuessDeps(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { buildContext, err := util.GetBuildContext() if err != nil { return nil, err } base := p.Get("dirname", ".").(string) skipImport := p.Get("skipImport", false).(bool) name := guessPackageName(buildContext, base) Info("Generating a YAML configuration file and guessing the dependencies") config := new(cfg.Config) // Get the name of the top level package config.Name = name // Import by looking at other package managers and looking over the // entire directory structure. // Attempt to import from other package managers. if !skipImport { Info("Attempting to import from other package managers (use --skip-import to skip)") deps := []*cfg.Dependency{} absBase, err := filepath.Abs(base) if err != nil { return nil, err } if d, ok := guessImportGodep(absBase); ok { Info("Importing Godep configuration") Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide") deps = d } else if d, ok := guessImportGPM(absBase); ok { Info("Importing GPM configuration") deps = d } else if d, ok := guessImportGB(absBase); ok { Info("Importing GB configuration") deps = d } for _, i := range deps { Info("Found imported reference to %s\n", i.Name) config.Imports = append(config.Imports, i) } } // Resolve dependencies by looking at the tree. r, err := dependency.NewResolver(base) if err != nil { return nil, err } h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}} r.Handler = h sortable, err := r.ResolveLocal(false) if err != nil { return nil, err } sort.Strings(sortable) vpath := r.VendorDir if !strings.HasSuffix(vpath, "/") { vpath = vpath + string(os.PathSeparator) } for _, pa := range sortable { n := strings.TrimPrefix(pa, vpath) root := util.GetRootFromPackage(n) if !config.HasDependency(root) { Info("Found reference to %s\n", n) d := &cfg.Dependency{ Name: root, } subpkg := strings.TrimPrefix(n, root) if len(subpkg) > 0 && subpkg != "/" { d.Subpackages = []string{subpkg} } config.Imports = append(config.Imports, d) } else { subpkg := strings.TrimPrefix(n, root) if len(subpkg) > 0 && subpkg != "/" { subpkg = strings.TrimPrefix(subpkg, "/") d := config.Imports.Get(root) f := false for _, v := range d.Subpackages { if v == subpkg { f = true } } if !f { Info("Adding sub-package %s to %s\n", subpkg, root) d.Subpackages = append(d.Subpackages, subpkg) } } } } return config, nil }
// addPkgsToConfig adds the given packages to the config file. // // Along the way it: // - ensures that this package is not in the ignore list // - checks to see if this is already in the dependency list. // - splits version of of package name and adds the version attribute // - seperates repo from packages // - sets up insecure repo URLs where necessary // - generates a list of subpackages func addPkgsToConfig(conf *cfg.Config, names []string, insecure bool) error { msg.Info("Preparing to install %d package.", len(names)) for _, name := range names { var version string parts := strings.Split(name, "#") if len(parts) > 1 { name = parts[0] version = parts[1] } root := util.GetRootFromPackage(name) if len(root) == 0 { return fmt.Errorf("Package name is required for %q.", name) } if conf.HasDependency(root) { // Check if the subpackage is present. subpkg := strings.TrimPrefix(name, root) subpkg = strings.TrimPrefix(subpkg, "/") if subpkg != "" { found := false dep := conf.Imports.Get(root) for _, s := range dep.Subpackages { if s == subpkg { found = true break } } if found { msg.Warn("Package %q is already in glide.yaml. Skipping", name) } else { dep.Subpackages = append(dep.Subpackages, subpkg) msg.Info("Adding sub-package %s to existing import %s", subpkg, root) } } else { msg.Warn("Package %q is already in glide.yaml. Skipping", root) } continue } if conf.HasIgnore(root) { msg.Warn("Package %q is set to be ignored in glide.yaml. Skipping", root) continue } dep := &cfg.Dependency{ Name: root, } if version != "" { dep.Reference = version } // When retriving from an insecure location set the repo to the // insecure location. if insecure { dep.Repository = "http://" + root } subpkg := strings.TrimPrefix(name, root) if len(subpkg) > 0 && subpkg != "/" { dep.Subpackages = []string{strings.TrimPrefix(subpkg, "/")} } if dep.Reference != "" { msg.Info("Importing %s with the version %s", dep.Name, dep.Reference) } else { msg.Info("Importing %s", dep.Name) } conf.Imports = append(conf.Imports, dep) } return nil }
// mergeGuess guesses dependencies and merges. // // This always returns true because it always handles the job of searching // for dependencies. So generally it should be the last merge strategy // that you try. func mergeGuess(dir, pkg string, f *flattening, scanned map[string]bool) ([]string, bool) { deps := f.deps Info("Scanning %s for dependencies.", pkg) buildContext, err := util.GetBuildContext() if err != nil { Warn("Could not scan package %q: %s", pkg, err) return []string{}, false } res := []string{} if _, err := os.Stat(dir); err != nil { Warn("Directory is missing: %s", dir) return res, true } d := walkDeps(buildContext, dir, pkg) for _, oname := range d { if _, ok := scanned[oname]; ok { //Info("===> Scanned %s already. Skipping", name) continue } Debug("=> Scanning %s", oname) name, _ := NormalizeName(oname) //if _, ok := deps[name]; ok { //scanned[oname] = true //Debug("====> Seen %s already. Skipping", name) //continue //} if f.conf.HasIgnore(name) { Debug("==> Skipping %s because it is on the ignore list", name) continue } found := findPkg(buildContext, name, dir) switch found.PType { case ptypeUnknown: Info("==> Unknown %s (%s)", name, oname) Debug("✨☆ Undownloaded dependency: %s", name) repo := util.GetRootFromPackage(name) nd := &cfg.Dependency{ Name: name, Repository: "https://" + repo, } deps[name] = nd res = append(res, name) case ptypeGoroot, ptypeCgo: scanned[oname] = true // Why do we break rather than continue? break default: // We're looking for dependencies that might exist in $GOPATH // but not be on vendor. We add any that are on $GOPATH. if _, ok := deps[name]; !ok { Debug("✨☆ GOPATH dependency: %s", name) nd := &cfg.Dependency{Name: name} deps[name] = nd res = append(res, name) } scanned[oname] = true } } return res, true }