// loadProgram loads the specified set of packages (plus their tests)
// and all their dependencies, from source, through the specified build
// context. Only packages in pkgs will have their functions bodies typechecked.
func loadProgram(ctxt *build.Context, pkgs map[string]bool) (*loader.Program, error) {
conf := loader.Config{
Build: ctxt,
SourceImports: true,
ParserMode: parser.ParseComments,
// TODO(adonovan): enable this. Requires making a lot of code more robust!
AllowErrors: false,
}
// Optimization: don't type-check the bodies of functions in our
// dependencies, since we only need exported package members.
conf.TypeCheckFuncBodies = func(p string) bool {
return pkgs[p] || pkgs[strings.TrimSuffix(p, "_test")]
}
if Verbose {
var list []string
for pkg := range pkgs {
list = append(list, pkg)
}
sort.Strings(list)
for _, pkg := range list {
fmt.Fprintf(os.Stderr, "Loading package: %s\n", pkg)
}
}
for pkg := range pkgs {
if err := conf.ImportWithTests(pkg); err != nil {
return nil, err
}
}
return conf.Load()
}
func (p *Parser) Parse(path string) error {
dir := filepath.Dir(path)
files, err := ioutil.ReadDir(dir)
if err != nil {
fmt.Printf("// [ERROR] Parse(%s) -> ioutil.ReadDir(%#v) -> err=<%#v>\n", path, dir, err)
return err
}
var astFiles []*ast.File
var conf loader.Config
conf.TypeCheckFuncBodies = func(_ string) bool { return false }
conf.TypeChecker.DisableUnusedImportCheck = true
conf.TypeChecker.Importer = importer.Default()
for _, fi := range files {
if filepath.Ext(fi.Name()) != ".go" {
continue
}
fpath := filepath.Join(dir, fi.Name())
f, err := conf.ParseFile(fpath, nil)
if err != nil {
fmt.Printf("// [ERROR] Parse(%s) -> conf.ParseFile(%#v) -> err=<%#v>\n", path, fpath, err)
return err
}
if fi.Name() == filepath.Base(path) {
p.file = f
}
astFiles = append(astFiles, f)
}
abs, err := filepath.Abs(path)
if err != nil {
fmt.Printf("// [ERROR] Parse(%s) -> filepath.Abs(%#v) -> err=<%#v>\n", path, path, err)
return err
}
// Type-check a package consisting of this file.
// Type information for the imported packages
// comes from $GOROOT/pkg/$GOOS_$GOOARCH/fmt.a.
conf.CreateFromFiles(abs, astFiles...)
prog, err := conf.Load()
if err != nil {
fmt.Printf("// [ERROR] Parse(%s) -> conf.Load() -> err=<%#v>\n", path, err)
return err
} else if len(prog.Created) != 1 {
panic("expected only one Created package")
}
p.path = abs
p.pkg = prog.Created[0].Pkg
return nil
}
// reduceScope is called for one-shot queries that need only a single
// typed package. It attempts to guess the query package from pos and
// reduce the analysis scope (set of loaded packages) to just that one
// plus (the exported parts of) its dependencies. It leaves its
// arguments unchanged on failure.
//
// TODO(adonovan): this is a real mess... but it's fast.
//
func reduceScope(pos string, conf *loader.Config) {
fqpos, err := fastQueryPos(pos)
if err != nil {
return // bad query
}
// TODO(adonovan): fix: this gives the wrong results for files
// in non-importable packages such as tests and ad-hoc packages
// specified as a list of files (incl. the oracle's tests).
_, importPath, err := guessImportPath(fqpos.fset.File(fqpos.start).Name(), conf.Build)
if err != nil {
return // can't find GOPATH dir
}
if importPath == "" {
return
}
// Check that it's possible to load the queried package.
// (e.g. oracle tests contain different 'package' decls in same dir.)
// Keep consistent with logic in loader/util.go!
cfg2 := *conf.Build
cfg2.CgoEnabled = false
bp, err := cfg2.Import(importPath, "", 0)
if err != nil {
return // no files for package
}
// Check that the queried file appears in the package:
// it might be a '// +build ignore' from an ad-hoc main
// package, e.g. $GOROOT/src/net/http/triv.go.
if !pkgContainsFile(bp, fqpos.fset.File(fqpos.start).Name()) {
return // not found
}
conf.TypeCheckFuncBodies = func(p string) bool { return p == importPath }
// Ignore packages specified on command line.
conf.CreatePkgs = nil
conf.ImportPkgs = nil
// Instead load just the one containing the query position
// (and possibly its corresponding tests/production code).
// TODO(adonovan): set 'augment' based on which file list
// contains
_ = conf.ImportWithTests(importPath) // ignore error
}
// importQueryPackage finds the package P containing the
// query position and tells conf to import it.
// It returns the package's path.
func importQueryPackage(pos string, conf *loader.Config) (string, error) {
fqpos, err := fastQueryPos(pos)
if err != nil {
return "", err // bad query
}
filename := fqpos.fset.File(fqpos.start).Name()
// This will not work for ad-hoc packages
// such as $GOROOT/src/net/http/triv.go.
// TODO(adonovan): ensure we report a clear error.
_, importPath, err := guessImportPath(filename, conf.Build)
if err != nil {
return "", err // can't find GOPATH dir
}
if importPath == "" {
return "", fmt.Errorf("can't guess import path from %s", filename)
}
// Check that it's possible to load the queried package.
// (e.g. oracle tests contain different 'package' decls in same dir.)
// Keep consistent with logic in loader/util.go!
cfg2 := *conf.Build
cfg2.CgoEnabled = false
bp, err := cfg2.Import(importPath, "", 0)
if err != nil {
return "", err // no files for package
}
switch pkgContainsFile(bp, filename) {
case 'T':
conf.ImportWithTests(importPath)
case 'X':
conf.ImportWithTests(importPath)
importPath += "_test" // for TypeCheckFuncBodies
case 'G':
conf.Import(importPath)
default:
return "", fmt.Errorf("package %q doesn't contain file %s",
importPath, filename)
}
conf.TypeCheckFuncBodies = func(p string) bool { return p == importPath }
return importPath, nil
}
// importQueryPackage finds the package P containing the
// query position and tells conf to import it.
// It returns the package's path.
func importQueryPackage(pos string, conf *loader.Config) (string, error) {
fqpos, err := fastQueryPos(conf.Build, pos)
if err != nil {
return "", err // bad query
}
filename := fqpos.fset.File(fqpos.start).Name()
_, importPath, err := guessImportPath(filename, conf.Build)
if err != nil {
// Can't find GOPATH dir.
// Treat the query file as its own package.
importPath = "command-line-arguments"
conf.CreateFromFilenames(importPath, filename)
} else {
// Check that it's possible to load the queried package.
// (e.g. guru tests contain different 'package' decls in same dir.)
// Keep consistent with logic in loader/util.go!
cfg2 := *conf.Build
cfg2.CgoEnabled = false
bp, err := cfg2.Import(importPath, "", 0)
if err != nil {
return "", err // no files for package
}
switch pkgContainsFile(bp, filename) {
case 'T':
conf.ImportWithTests(importPath)
case 'X':
conf.ImportWithTests(importPath)
importPath += "_test" // for TypeCheckFuncBodies
case 'G':
conf.Import(importPath)
default:
// This happens for ad-hoc packages like
// $GOROOT/src/net/http/triv.go.
return "", fmt.Errorf("package %q doesn't contain file %s",
importPath, filename)
}
}
conf.TypeCheckFuncBodies = func(p string) bool { return p == importPath }
return importPath, nil
}
func (u *Unexporter) loadProgram(pkgPath string) (err error) {
wd, err := os.Getwd()
if err != nil {
return err
}
bpkg, err := u.ctxt.Import(pkgPath, wd, build.ImportComment)
if err != nil {
return err
}
_, rev, _ := importgraph.Build(u.ctxt)
pkgs := rev.Search(bpkg.ImportPath)
conf := loader.Config{
Build: u.ctxt,
ParserMode: parser.ParseComments,
AllowErrors: false,
}
// Optimization: don't type-check the bodies of functions in our
// dependencies, since we only need exported package members.
conf.TypeCheckFuncBodies = func(p string) bool {
return pkgs[p] || pkgs[strings.TrimSuffix(p, "_test")]
}
for pkg := range pkgs {
conf.ImportWithTests(pkg)
}
u.prog, err = conf.Load()
if err != nil {
return
}
for p, info := range u.prog.AllPackages {
if p.Path() == bpkg.ImportPath {
u.pkgInfo = info
break
}
}
return
}
// globalReferrers reports references throughout the entire workspace to the
// object at the specified source position. Its defining package is defpkg,
// and the query package is qpkg. isPkgLevel indicates whether the object
// is defined at package-level.
func globalReferrers(q *Query, qpkg, defpkg string, objposn token.Position, isPkgLevel bool) error {
// Scan the workspace and build the import graph.
// Ignore broken packages.
_, rev, _ := importgraph.Build(q.Build)
// Find the set of packages that depend on defpkg.
// Only function bodies in those packages need type-checking.
var users map[string]bool
if isPkgLevel {
users = rev[defpkg] // direct importers
if users == nil {
users = make(map[string]bool)
}
users[defpkg] = true // plus the defining package itself
} else {
users = rev.Search(defpkg) // transitive importers
}
// Prepare to load the larger program.
fset := token.NewFileSet()
lconf := loader.Config{
Fset: fset,
Build: q.Build,
TypeCheckFuncBodies: func(p string) bool {
return users[strings.TrimSuffix(p, "_test")]
},
}
allowErrors(&lconf)
// The importgraph doesn't treat external test packages
// as separate nodes, so we must use ImportWithTests.
for path := range users {
lconf.ImportWithTests(path)
}
// The remainder of this function is somewhat tricky because it
// operates on the concurrent stream of packages observed by the
// loader's AfterTypeCheck hook. Most of guru's helper
// functions assume the entire program has already been loaded,
// so we can't use them here.
// TODO(adonovan): smooth things out once the other changes have landed.
// Results are reported concurrently from within the
// AfterTypeCheck hook. The program may provide a useful stream
// of information even if the user doesn't let the program run
// to completion.
var (
mu sync.Mutex
qobj types.Object
qinfo *loader.PackageInfo // info for qpkg
)
// For efficiency, we scan each package for references
// just after it has been type-checked. The loader calls
// AfterTypeCheck (concurrently), providing us with a stream of
// packages.
lconf.AfterTypeCheck = func(info *loader.PackageInfo, files []*ast.File) {
// AfterTypeCheck may be called twice for the same package due to augmentation.
// Only inspect packages that depend on the declaring package
// (and thus were type-checked).
if lconf.TypeCheckFuncBodies(info.Pkg.Path()) {
// Record the query object and its package when we see it.
mu.Lock()
if qobj == nil && info.Pkg.Path() == defpkg {
// Find the object by its position (slightly ugly).
qobj = findObject(fset, &info.Info, objposn)
if qobj == nil {
// It really ought to be there;
// we found it once already.
log.Fatalf("object at %s not found in package %s",
objposn, defpkg)
}
// Object found.
qinfo = info
q.Output(fset, &referrersInitialResult{
qinfo: qinfo,
obj: qobj,
})
}
obj := qobj
mu.Unlock()
// Look for references to the query object.
if obj != nil {
outputUses(q, fset, usesOf(obj, info), info.Pkg)
}
}
clearInfoFields(info) // save memory
}
lconf.Load() // ignore error
if qobj == nil {
log.Fatal("query object not found during reloading")
}
return nil // success
}
// packageReferrers reports all references to the specified package
// throughout the workspace.
func packageReferrers(q *Query, path string) error {
// Scan the workspace and build the import graph.
// Ignore broken packages.
_, rev, _ := importgraph.Build(q.Build)
// Find the set of packages that directly import the query package.
// Only those packages need typechecking of function bodies.
users := rev[path]
// Load the larger program.
fset := token.NewFileSet()
lconf := loader.Config{
Fset: fset,
Build: q.Build,
TypeCheckFuncBodies: func(p string) bool {
return users[strings.TrimSuffix(p, "_test")]
},
}
allowErrors(&lconf)
// The importgraph doesn't treat external test packages
// as separate nodes, so we must use ImportWithTests.
for path := range users {
lconf.ImportWithTests(path)
}
// Subtle! AfterTypeCheck needs no mutex for qpkg because the
// topological import order gives us the necessary happens-before edges.
// TODO(adonovan): what about import cycles?
var qpkg *types.Package
// For efficiency, we scan each package for references
// just after it has been type-checked. The loader calls
// AfterTypeCheck (concurrently), providing us with a stream of
// packages.
lconf.AfterTypeCheck = func(info *loader.PackageInfo, files []*ast.File) {
// AfterTypeCheck may be called twice for the same package due to augmentation.
if info.Pkg.Path() == path && qpkg == nil {
// Found the package of interest.
qpkg = info.Pkg
fakepkgname := types.NewPkgName(token.NoPos, qpkg, qpkg.Name(), qpkg)
q.Output(fset, &referrersInitialResult{
qinfo: info,
obj: fakepkgname, // bogus
})
}
// Only inspect packages that directly import the
// declaring package (and thus were type-checked).
if lconf.TypeCheckFuncBodies(info.Pkg.Path()) {
// Find PkgNames that refer to qpkg.
// TODO(adonovan): perhaps more useful would be to show imports
// of the package instead of qualified identifiers.
var refs []*ast.Ident
for id, obj := range info.Uses {
if obj, ok := obj.(*types.PkgName); ok && obj.Imported() == qpkg {
refs = append(refs, id)
}
}
outputUses(q, fset, refs, info.Pkg)
}
clearInfoFields(info) // save memory
}
lconf.Load() // ignore error
if qpkg == nil {
log.Fatalf("query package %q not found during reloading", path)
}
return nil
}
func bundle(w io.Writer, initialPkg, dest, prefix string) error {
// Load the initial package.
conf := loader.Config{ParserMode: parser.ParseComments, Build: ctxt}
conf.TypeCheckFuncBodies = func(p string) bool { return p == initialPkg }
conf.Import(initialPkg)
lprog, err := conf.Load()
if err != nil {
log.Fatal(err)
}
info := lprog.Package(initialPkg)
objsToUpdate := make(map[types.Object]bool)
var rename func(from types.Object)
rename = func(from types.Object) {
if !objsToUpdate[from] {
objsToUpdate[from] = true
// Renaming a type that is used as an embedded field
// requires renaming the field too. e.g.
// type T int // if we rename this to U..
// var s struct {T}
// print(s.T) // ...this must change too
if _, ok := from.(*types.TypeName); ok {
for id, obj := range info.Uses {
if obj == from {
if field := info.Defs[id]; field != nil {
rename(field)
}
}
}
}
}
}
// Rename each package-level object.
scope := info.Pkg.Scope()
for _, name := range scope.Names() {
rename(scope.Lookup(name))
}
var out bytes.Buffer
fmt.Fprintf(&out, "// Code generated by golang.org/x/tools/cmd/bundle command:\n")
fmt.Fprintf(&out, "// $ bundle %s %s %s\n\n", initialPkg, dest, prefix)
// Concatenate package comments from of all files.
for _, f := range info.Files {
if doc := f.Doc.Text(); strings.TrimSpace(doc) != "" {
for _, line := range strings.Split(doc, "\n") {
fmt.Fprintf(&out, "// %s\n", line)
}
}
}
// TODO(adonovan): don't assume pkg.name == basename(pkg.path).
fmt.Fprintf(&out, "package %s\n\n", filepath.Base(dest))
// Print a single declaration that imports all necessary packages.
// TODO(adonovan):
// - support renaming imports.
// - preserve comments from the original import declarations.
for _, f := range info.Files {
for _, imp := range f.Imports {
if imp.Name != nil {
log.Fatalf("%s: renaming imports not supported",
lprog.Fset.Position(imp.Pos()))
}
}
}
fmt.Fprintln(&out, "import (")
for _, p := range info.Pkg.Imports() {
if p.Path() == dest {
continue
}
fmt.Fprintf(&out, "\t%q\n", p.Path())
}
fmt.Fprintln(&out, ")\n")
// Modify and print each file.
for _, f := range info.Files {
// Update renamed identifiers.
for id, obj := range info.Defs {
if objsToUpdate[obj] {
id.Name = prefix + obj.Name()
}
}
for id, obj := range info.Uses {
if objsToUpdate[obj] {
id.Name = prefix + obj.Name()
}
}
// For each qualified identifier that refers to the
// destination package, remove the qualifier.
// The "@@@." strings are removed in postprocessing.
ast.Inspect(f, func(n ast.Node) bool {
if sel, ok := n.(*ast.SelectorExpr); ok {
if id, ok := sel.X.(*ast.Ident); ok {
if obj, ok := info.Uses[id].(*types.PkgName); ok {
if obj.Imported().Path() == dest {
id.Name = "@@@"
}
}
}
}
return true
})
// Pretty-print package-level declarations.
// but no package or import declarations.
//
// TODO(adonovan): this may cause loss of comments
// preceding or associated with the package or import
// declarations or not associated with any declaration.
// Check.
var buf bytes.Buffer
for _, decl := range f.Decls {
if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT {
continue
}
buf.Reset()
format.Node(&buf, lprog.Fset, decl)
// Remove each "@@@." in the output.
// TODO(adonovan): not hygienic.
out.Write(bytes.Replace(buf.Bytes(), []byte("@@@."), nil, -1))
out.WriteString("\n\n")
}
}
// Now format the entire thing.
result, err := format.Source(out.Bytes())
if err != nil {
log.Fatalf("formatting failed: %v", err)
}
_, err = w.Write(result)
return err
}
func bundle(src, dst, dstpkg, prefix string) ([]byte, error) {
// Load the initial package.
conf := loader.Config{ParserMode: parser.ParseComments, Build: ctxt}
conf.TypeCheckFuncBodies = func(p string) bool { return p == src }
conf.Import(src)
lprog, err := conf.Load()
if err != nil {
return nil, err
}
info := lprog.Package(src)
if prefix == "" {
pkgName := info.Files[0].Name.Name
prefix = pkgName + "_"
}
objsToUpdate := make(map[types.Object]bool)
var rename func(from types.Object)
rename = func(from types.Object) {
if !objsToUpdate[from] {
objsToUpdate[from] = true
// Renaming a type that is used as an embedded field
// requires renaming the field too. e.g.
// type T int // if we rename this to U..
// var s struct {T}
// print(s.T) // ...this must change too
if _, ok := from.(*types.TypeName); ok {
for id, obj := range info.Uses {
if obj == from {
if field := info.Defs[id]; field != nil {
rename(field)
}
}
}
}
}
}
// Rename each package-level object.
scope := info.Pkg.Scope()
for _, name := range scope.Names() {
rename(scope.Lookup(name))
}
var out bytes.Buffer
fmt.Fprintf(&out, "// Code generated by golang.org/x/tools/cmd/bundle.\n")
if *outputFile != "" {
fmt.Fprintf(&out, "//go:generate bundle %s\n", strings.Join(os.Args[1:], " "))
} else {
fmt.Fprintf(&out, "// $ bundle %s\n", strings.Join(os.Args[1:], " "))
}
fmt.Fprintf(&out, "\n")
// Concatenate package comments from all files...
for _, f := range info.Files {
if doc := f.Doc.Text(); strings.TrimSpace(doc) != "" {
for _, line := range strings.Split(doc, "\n") {
fmt.Fprintf(&out, "// %s\n", line)
}
}
}
// ...but don't let them become the actual package comment.
fmt.Fprintln(&out)
fmt.Fprintf(&out, "package %s\n\n", dstpkg)
// BUG(adonovan,shurcooL): bundle may generate incorrect code
// due to shadowing between identifiers and imported package names.
//
// The generated code will either fail to compile or
// (unlikely) compile successfully but have different behavior
// than the original package. The risk of this happening is higher
// when the original package has renamed imports (they're typically
// renamed in order to resolve a shadow inside that particular .go file).
// TODO(adonovan,shurcooL):
// - detect shadowing issues, and either return error or resolve them
// - preserve comments from the original import declarations.
// pkgStd and pkgExt are sets of printed import specs. This is done
// to deduplicate instances of the same import name and path.
var pkgStd = make(map[string]bool)
var pkgExt = make(map[string]bool)
for _, f := range info.Files {
for _, imp := range f.Imports {
path, err := strconv.Unquote(imp.Path.Value)
if err != nil {
log.Fatalf("invalid import path string: %v", err) // Shouldn't happen here since conf.Load succeeded.
}
if path == dst {
continue
}
if newPath, ok := importMap[path]; ok {
path = newPath
}
var name string
if imp.Name != nil {
name = imp.Name.Name
}
spec := fmt.Sprintf("%s %q", name, path)
if isStandardImportPath(path) {
pkgStd[spec] = true
} else {
if *underscore {
spec = strings.Replace(spec, "golang.org/", "golang_org/", 1)
}
pkgExt[spec] = true
}
}
}
// Print a single declaration that imports all necessary packages.
fmt.Fprintln(&out, "import (")
for p := range pkgStd {
fmt.Fprintf(&out, "\t%s\n", p)
}
if len(pkgExt) > 0 {
fmt.Fprintln(&out)
}
for p := range pkgExt {
fmt.Fprintf(&out, "\t%s\n", p)
}
fmt.Fprint(&out, ")\n\n")
// Modify and print each file.
for _, f := range info.Files {
// Update renamed identifiers.
for id, obj := range info.Defs {
if objsToUpdate[obj] {
id.Name = prefix + obj.Name()
}
}
for id, obj := range info.Uses {
if objsToUpdate[obj] {
id.Name = prefix + obj.Name()
}
}
// For each qualified identifier that refers to the
// destination package, remove the qualifier.
// The "@@@." strings are removed in postprocessing.
ast.Inspect(f, func(n ast.Node) bool {
if sel, ok := n.(*ast.SelectorExpr); ok {
if id, ok := sel.X.(*ast.Ident); ok {
if obj, ok := info.Uses[id].(*types.PkgName); ok {
if obj.Imported().Path() == dst {
id.Name = "@@@"
}
}
}
}
return true
})
// Pretty-print package-level declarations.
// but no package or import declarations.
//
// TODO(adonovan): this may cause loss of comments
// preceding or associated with the package or import
// declarations or not associated with any declaration.
// Check.
var buf bytes.Buffer
for _, decl := range f.Decls {
if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT {
continue
}
buf.Reset()
format.Node(&buf, lprog.Fset, decl)
// Remove each "@@@." in the output.
// TODO(adonovan): not hygienic.
out.Write(bytes.Replace(buf.Bytes(), []byte("@@@."), nil, -1))
out.WriteString("\n\n")
}
}
// Now format the entire thing.
result, err := format.Source(out.Bytes())
if err != nil {
log.Fatalf("formatting failed: %v", err)
}
return result, nil
}