func downloadDependency(dep vendor.Dependency, errors *uint32, vendorDir string, recursive bool) error {
extraMsg := ""
if !dep.NoTests {
extraMsg = "(including tests)"
}
if dep.AllFiles {
extraMsg = "(without file exclusions)"
}
if recursive {
log.Printf("fetching recursive %s %s", dep.Importpath, extraMsg)
} else {
log.Printf("fetching %s %s", dep.Importpath, extraMsg)
}
repo, err := vendor.NewRemoteRepo(dep.Repository, dep.VCS, rbInsecure)
if err != nil {
return fmt.Errorf("dependency could not be processed: %s", err)
}
// We can't pass the branch here, and benefit from narrow clones, as the
// revision might not be in the branch tree anymore. Thanks rebase.
wc, err := GlobalDownloader.Get(repo, "", "", dep.Revision)
if err != nil {
return fmt.Errorf("dependency could not be fetched: %s", err)
}
dst := filepath.Join(vendorDir, dep.Importpath)
src := filepath.Join(wc.Dir(), dep.Path)
if _, err := os.Stat(dst); err == nil {
if err := fileutils.RemoveAll(dst); err != nil {
return fmt.Errorf("dependency could not be deleted: %v", err)
}
}
if err := fileutils.Copypath(dst, src, !dep.NoTests, dep.AllFiles); err != nil {
return err
}
if err := fileutils.CopyLicense(dst, wc.Dir()); err != nil {
return err
}
// Check for for manifests in dependencies
man := filepath.Join(dst, "vendor", "manifest")
venDir := filepath.Join(dst, "vendor")
if _, err := os.Stat(man); err == nil {
m, err := vendor.ReadManifest(man)
if err != nil {
return fmt.Errorf("could not load manifest: %v", err)
}
for _, d := range m.Dependencies {
if err := downloadDependency(d, errors, venDir, true); err != nil {
log.Printf("%s: %v", d.Importpath, err)
atomic.AddUint32(errors, 1)
}
}
}
return nil
}
func fetchRecursive(m *vendor.Manifest, fullPath string, level int) error {
path := stripscheme(fullPath)
// Don't even bother the user about skipping packages we just fetched
for _, p := range fetchedToday {
if contains(p, path) {
return nil
}
}
// First, check if this or a parent is already vendored
if m.HasImportpath(path) {
if level == 0 {
return fmt.Errorf("%s or a parent of it is already vendored", path)
} else {
// TODO: print a different message for packages fetched during this session
logIndent(level, "Skipping (existing):", path)
return nil
}
}
// Next, check if we are trying to vendor from the same repository we are in
if importPath != "" && contains(importPath, path) {
if level == 0 {
return fmt.Errorf("refusing to vendor a subpackage of \".\"")
} else {
logIndent(level, "Skipping (subpackage of \".\"):", path)
return nil
}
}
if level == 0 {
log.Println("Fetching:", path)
} else {
logIndent(level, "Fetching recursive dependency:", path)
}
// Finally, check if we already vendored a subpackage and remove it
for _, subp := range m.GetSubpackages(path) {
if !contains(subp.Importpath, fetchRoot) { // ignore parents of the root
ignore := false
for _, d := range fetchedToday {
if contains(d, subp.Importpath) {
ignore = true // No need to warn the user if we just downloaded it
}
}
if !ignore {
logIndent(level, "Deleting existing subpackage to prevent overlap:", subp.Importpath)
}
}
if err := m.RemoveDependency(subp); err != nil {
return fmt.Errorf("failed to remove subpackage: %v", err)
}
}
if err := fileutils.RemoveAll(filepath.Join(vendorDir, path)); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove existing folder: %v", err)
}
// Find and download the repository
repo, extra, err := GlobalDownloader.DeduceRemoteRepo(fullPath, insecure)
if err != nil {
return err
}
if level == 0 {
rootRepoURL = repo.URL()
}
var wc vendor.WorkingCopy
if repo.URL() == rootRepoURL {
wc, err = GlobalDownloader.Get(repo, branch, tag, revision)
} else {
wc, err = GlobalDownloader.Get(repo, "", "", "")
}
if err != nil {
return err
}
// Add the dependency to the manifest
rev, err := wc.Revision()
if err != nil {
return err
}
b, err := wc.Branch()
if err != nil {
return err
}
dep := vendor.Dependency{
Importpath: path,
Repository: repo.URL(),
VCS: repo.Type(),
Revision: rev,
Branch: b,
Path: extra,
NoTests: !tests,
AllFiles: all,
}
if err := m.AddDependency(dep); err != nil {
return err
}
// Copy the code to the vendor folder
dst := filepath.Join(vendorDir, dep.Importpath)
src := filepath.Join(wc.Dir(), dep.Path)
if err := fileutils.Copypath(dst, src, !dep.NoTests, dep.AllFiles); err != nil {
return err
}
if err := fileutils.CopyLicense(dst, wc.Dir()); err != nil {
return err
}
if err := vendor.WriteManifest(manifestFile, m); err != nil {
return err
}
// Recurse
fetchedToday = append(fetchedToday, path)
if !noRecurse {
// Look for dependencies in src, not going past wc.Dir() when looking for /vendor/,
// knowing that wc.Dir() corresponds to rootRepoPath
if !strings.HasSuffix(dep.Importpath, dep.Path) {
return fmt.Errorf("unable to derive the root repo import path")
}
rootRepoPath := strings.TrimRight(strings.TrimSuffix(dep.Importpath, dep.Path), "/")
deps, err := vendor.ParseImports(src, wc.Dir(), rootRepoPath, tests, all)
if err != nil {
return fmt.Errorf("failed to parse imports: %s", err)
}
for d := range deps {
if strings.Index(d, ".") == -1 { // TODO: replace this silly heuristic
continue
}
if err := fetchRecursive(m, d, level+1); err != nil {
if strings.HasPrefix(err.Error(), "error fetching") { // I know, ok?
return err
} else {
return fmt.Errorf("error fetching %s: %s", d, err)
}
}
}
}
return nil
}
Revision: rev,
Branch: branch,
Path: d.Path,
NoTests: d.NoTests,
AllFiles: d.AllFiles,
}
if err := fileutils.RemoveAll(filepath.Join(vendorDir, filepath.FromSlash(d.Importpath))); err != nil {
// TODO(dfc) need to apply vendor.cleanpath here to remove intermediate directories.
return fmt.Errorf("dependency could not be deleted: %v", err)
}
dst := filepath.Join(vendorDir, filepath.FromSlash(dep.Importpath))
src := filepath.Join(wc.Dir(), dep.Path)
if err := fileutils.Copypath(dst, src, !d.NoTests, d.AllFiles); err != nil {
return err
}
if err := fileutils.CopyLicense(dst, wc.Dir()); err != nil {
return err
}
if err := m.AddDependency(dep); err != nil {
return err
}
if err := vendor.WriteManifest(manifestFile, m); err != nil {
return err
}
}