func TestParsePackage(t *testing.T) {
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, "test", nil, parser.ParseComments)
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 1 {
t.Fatalf("Expected len(pkgs) == 1, got %d", len(pkgs))
}
comments := parsePackage(pkgs["test"])
expected := []string{
"A comment inside test.go\n",
"More restructured text, in doc.go\n",
"Here's a comment in a.go\n",
"An interesting\nmulti-line\ncomment inside\nz.go\n",
}
if len(comments) != len(expected) {
t.Fatalf("Expected %#v, got %#v", expected, comments)
}
for i := 0; i < len(comments); i++ {
if comments[i] != expected[i] {
t.Fatalf("Expected %s, got %s", expected, comments)
}
}
}
// getPageInfo returns the PageInfo for a package directory path. If the
// parameter try is true, no errors are logged if getPageInfo fails.
// If there is no corresponding package in the directory,
// PageInfo.PDoc is nil. If there are no subdirectories,
// PageInfo.Dirs is nil.
//
func (h *httpHandler) getPageInfo(path string, try bool) PageInfo {
// the path is relative to h.fsroot
dirname := pathutil.Join(h.fsRoot, path)
// the package name is the directory name within its parent
// (use dirname instead of path because dirname is clean; i.e. has no trailing '/')
_, pkgname := pathutil.Split(dirname)
// filter function to select the desired .go files
filter := func(d *os.Dir) bool {
if isPkgFile(d) {
// Some directories contain main packages: Only accept
// files that belong to the expected package so that
// parser.ParsePackage doesn't return "multiple packages
// found" errors.
// Additionally, accept the special package name
// fakePkgName if we are looking at cmd documentation.
name := pkgName(dirname + "/" + d.Name)
return name == pkgname || h.fsRoot == *cmdroot && name == fakePkgName
}
return false
}
// get package AST
pkgs, err := parser.ParseDir(dirname, filter, parser.ParseComments)
if err != nil && !try {
// TODO: errors should be shown instead of an empty directory
log.Stderrf("parser.parseDir: %s", err)
}
if len(pkgs) != 1 && !try {
// TODO: should handle multiple packages
log.Stderrf("parser.parseDir: found %d packages", len(pkgs))
}
var pkg *ast.Package
for _, pkg = range pkgs {
break // take the first package found
}
// compute package documentation
var pdoc *doc.PackageDoc
if pkg != nil {
ast.PackageExports(pkg)
pdoc = doc.NewPackageDoc(pkg, pathutil.Clean(path)) // no trailing '/' in importpath
}
// get directory information
var dir *Directory
if tree, _ := fsTree.get(); tree != nil {
// directory tree is present; lookup respective directory
// (may still fail if the file system was updated and the
// new directory tree has not yet beet computed)
dir = tree.(*Directory).lookup(dirname)
} else {
// no directory tree present (either early after startup
// or command-line mode); compute one level for this page
dir = newDirectory(dirname, 1)
}
return PageInfo{pdoc, dir.listing(true), h.isPkg}
}
func parserPkg(pkgRealpath, pkgpath string) error {
if !compareFile(pkgRealpath) {
Info(pkgRealpath + " don't has updated")
return nil
}
fileSet := token.NewFileSet()
astPkgs, err := parser.ParseDir(fileSet, pkgRealpath, func(info os.FileInfo) bool {
name := info.Name()
return !info.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
}, parser.ParseComments)
if err != nil {
return err
}
for _, pkg := range astPkgs {
for _, fl := range pkg.Files {
for _, d := range fl.Decls {
switch specDecl := d.(type) {
case *ast.FuncDecl:
parserComments(specDecl.Doc, specDecl.Name.String(), fmt.Sprint(specDecl.Recv.List[0].Type.(*ast.StarExpr).X), pkgpath)
}
}
}
}
genRouterCode()
savetoFile(pkgRealpath)
return nil
}
func packageComment(pkg, pkgpath string) (info string, err os.Error) {
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments)
if err != nil {
return
}
for name := range pkgs {
if name == "main" {
continue
}
if info != "" {
return "", os.NewError("multiple non-main package docs")
}
pdoc := doc.NewPackageDoc(pkgs[name], pkg)
info = pdoc.Doc
}
// grab only first paragraph
if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 {
info = parts[0]
}
// replace newlines with spaces
info = strings.Replace(info, "\n", " ", -1)
// truncate
if len(info) > MaxCommentLength {
info = info[:MaxCommentLength]
}
return
}
func parseDir(dirpath string) map[string]*ast.Package {
// the package name is the directory name within its parent.
// (use dirname instead of path because dirname is clean; it
// has no trailing '/')
_, pkgname := path.Split(dirpath)
// filter function to select the desired .go files
filter := func(d os.FileInfo) bool {
if isPkgFile(d) {
// Some directories contain main packages: Only accept
// files that belong to the expected package so that
// parser.ParsePackage doesn't return "multiple packages
// found" errors.
// Additionally, accept the special package name
// fakePkgName if we are looking at cmd documentation.
name := pkgName(dirpath + "/" + d.Name())
return name == pkgname
}
return false
}
// get package AST
pkgs, err := parser.ParseDir(token.NewFileSet(), dirpath, filter, parser.ParseComments)
if err != nil {
println("parse", dirpath, err.Error())
panic("go ParseDir fail: " + err.Error())
}
return pkgs
}
func rewriteDir(src, dest string) error {
fmt.Printf("Rewriting directory %s ——> %s\n", src, dest)
// Make destination directory if it doesn't exist
if err := os.MkdirAll(dest, 0755); err != nil {
fmt.Fprintf(os.Stderr, "Target directory cannot be created (%s).\n", err)
return err
}
// Parse source directory
fileSet := token.NewFileSet()
pkgs, err := parser.ParseDir(fileSet, src, FilterGoFiles, parser.ParseComments)
if err != nil {
fmt.Fprintf(os.Stderr, "Problem parsing directory (%s)\n", err)
return err
}
// In every package, rewrite every file
for _, pkg := range pkgs {
for _, fileFile := range pkg.Files {
RewriteFile(fileSet, fileFile)
position := fileSet.Position(fileFile.Package)
_, filename := path.Split(position.Filename)
fmt.Printf(" %s ==> %s\n", filename, path.Join(dest, filename))
if err = PrintToFile(path.Join(dest, filename), fileSet, fileFile); err != nil {
fmt.Fprintf(os.Stderr, "Problem determining source filename (%s)", err)
os.Exit(1)
}
}
}
return nil
}
func parsePkgDir(p string, fset *token.FileSet) (*ast.Package, error) {
fi, err := os.Stat(p)
if err != nil {
return nil, fmt.Errorf("unable to stat file %#v: %s", p, err)
}
if !fi.IsDir() {
return nil, fmt.Errorf("%#v is not a directory", p)
}
mp, err := parser.ParseDir(fset, p, nil, 0)
if err != nil {
return nil, fmt.Errorf("unable to parse package at %#v: %s", p, err)
}
if len(mp) != 1 {
var ps []string
for _, pkg := range mp {
ps = append(ps, pkg.Name)
}
return nil, fmt.Errorf("more than one package found in %#v: %s", p, strings.Join(ps, ","))
}
var pkg *ast.Package
for _, v := range mp {
pkg = v
}
return pkg, nil
}
func generateMock(o *options) {
fset := token.NewFileSet()
// package path can be a directory
stat, err := os.Stat(o.packagePath)
if err != nil {
fmt.Printf("Failed to access %s. %v", o.packagePath, err)
}
if stat.IsDir() {
pkgs, err := parser.ParseDir(fset, o.packagePath, func(fileinfo os.FileInfo) bool {
return fileinfo.Name() != o.outfile
}, 0)
if err != nil {
fmt.Printf("Failed to parse %s. %v", o.packagePath, err)
os.Exit(2)
}
// Look for the type in each of the files in the directory
for _, pkg := range pkgs {
if generateMockFromAst(o, pkg) {
return
}
}
} else {
p, err := parser.ParseFile(fset, o.packagePath, nil, 0)
if err != nil {
fmt.Printf("Failed to parse %s. %v", o.packagePath, err)
os.Exit(2)
}
generateMockFromAst(o, p)
}
}
func main() {
dirs := os.Args[1:]
fs = token.NewFileSet()
//dirs := parseInput(*dir)
fmt.Println(dirs)
for _, cd := range dirs {
fmt.Println(cd)
nodes, err := parser.ParseDir(fs, cd, sourceFilter, 0)
if err != nil {
continue
//fmt.Printf("Error while parsing files in \"%s\".: %s\n", cd, err)
}
for _, node := range nodes {
ast.Inspect(node, RootHandeler)
}
}
fmt.Println(imports)
imports.GetImports()
//printer.Fprint(os.Stdout, fs, node)
}
func (p *Parser) Parse(fname string, isDir bool) error {
var err error
if p.PkgPath, err = getPkgPath(fname, isDir); err != nil {
return err
}
fset := token.NewFileSet()
if isDir {
packages, err := parser.ParseDir(fset, fname, nil, parser.ParseComments)
if err != nil {
return err
}
for _, pckg := range packages {
ast.Walk(&visitor{Parser: p}, pckg)
}
} else {
f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
if err != nil {
return err
}
ast.Walk(&visitor{Parser: p}, f)
}
return nil
}
func GrokDir(dir string) {
pkgs, err := parser.ParseDir(fset, dir, FilterDotGo, parser.Mode(0))
if err != nil {
panic(Sprintf("ERROR <%q> IN DIR <%s>", err, dir))
}
// Find the subdir that follows /pkg/
subdir := AfterFirstDelim(dir, "/pkg/", "")
if len(subdir) == 0 {
panic("dir does not contain /pkg/ plus a tail: " + dir)
}
// Trim trailing /, if any.
if subdir[len(subdir)-1] == '/' {
subdir = subdir[:len(subdir)-1]
}
for pk, pv := range pkgs {
// For dirpkg, remove final piece of subdir
// and replace with stated package name.
// Often these are the same, but not always.
updir := BeforeFinalDelim(subdir, "/", "")
dirpkg := Cond(updir == "", pk, updir+"/"+pk)
Printf("#dirpkg %#v\n", dirpkg)
Printf("#pv %#v\n", pv)
for _, fv := range pv.Files {
for i, dcl := range fv.Decls {
doDecl(dcl, i, dirpkg)
}
}
}
}
func TestImportBug(t *testing.T) {
buildPkg, err := build.Import("github.com/netbrain/importbug/foo", "", build.ImportComment)
if err != nil {
t.Fatal(err)
}
fset := token.NewFileSet()
packages, err := parser.ParseDir(fset, buildPkg.Dir, nil, 0)
if err != nil {
t.Fatal(err)
}
info := &types.Info{
Defs: make(map[*ast.Ident]types.Object),
}
for pName, p := range packages {
files := make([]*ast.File, 0, len(p.Files))
for _, f := range p.Files {
files = append(files, f)
}
conf := &types.Config{
FakeImportC: true,
Importer: importer.Default(),
}
_, err := conf.Check(pName, fset, files, info)
if err != nil {
log.Fatal(err)
}
}
}
func (l *Locator) discoverTypes(location string) ([]TypeDiscovery, error) {
discoveries, found := l.cache[location]
if found {
return discoveries, nil
}
sourcePath, err := util.ImportToDir(location)
if err != nil {
return nil, err
}
pkgs, err := parser.ParseDir(token.NewFileSet(), sourcePath, nil, parser.AllErrors)
if err != nil {
return nil, err
}
discoveries = make([]TypeDiscovery, 0)
for _, pkg := range pkgs {
for _, file := range pkg.Files {
for spec := range util.EachTypeSpecificationInFile(file) {
discoveries = append(discoveries, TypeDiscovery{
Location: location,
File: file,
Spec: spec,
})
}
}
}
l.cache[location] = discoveries
return discoveries, nil
}
func NewContextDeclFromSrcPath(root string) (*ContextDecl, error) {
decl := &ContextDecl{PackageMap: make(map[string]*PackageDecl)}
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
return nil
}
if info.Name()[0] == '.' {
return filepath.SkipDir
}
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, path, nil, parser.ParseComments|parser.AllErrors)
if err != nil {
return err
}
pkg := pkgs[info.Name()]
if pkg == nil {
return nil
}
fullImportPath, err := filepath.Rel(root, path)
if err != nil {
return err
}
fullImportPath = strings.Replace(fullImportPath, "\\", "/", -1)
decl.PackageMap[fullImportPath] = NewPackageDeclFromAstPackage(pkg, fullImportPath)
return nil
})
if err != nil {
return nil, err
}
return decl, nil
}
func importPackage(path string) (*token.FileSet, *ast.Package, error) {
buildPkg, err := build.Import(path, ".", 0)
if err != nil {
return nil, nil, err
}
dir := buildPkg.Dir
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, dir, func(fi os.FileInfo) bool {
for _, ignored := range buildPkg.IgnoredGoFiles {
if fi.Name() == ignored {
return false
}
}
for _, gofile := range buildPkg.GoFiles { // GoFiles doesn't contain tests
if fi.Name() == gofile {
return true
}
}
// we can also parse buildPkg.CgoFiles
// however, due to oracle can't parse CgoFiles, we don't parse them
// so any cgo user should not contain type and interfaces in CgoFiles that is `import "C"` is declared
return false
}, 0)
if err != nil {
return nil, nil, err
}
if len(pkgs) != 1 {
return nil, nil, fmt.Errorf("package %s contains %d packges, it must be 1", path, len(pkgs))
}
for _, pkg := range pkgs {
return fset, pkg, nil
}
panic("unreachable code")
}
func getPageInfo(pkgName, diskPath string) (pi PageInfo, err error) {
bpkg, err := build.ImportDir(diskPath, 0)
if err != nil {
return
}
inSet := make(map[string]bool)
for _, name := range bpkg.GoFiles {
inSet[filepath.Base(name)] = true
}
pi.FSet = token.NewFileSet()
filter := func(fi os.FileInfo) bool {
return inSet[fi.Name()]
}
aPkgMap, err := parser.ParseDir(pi.FSet, diskPath, filter, 0)
if err != nil {
return
}
aPkg := aPkgMap[pathpkg.Base(pkgName)]
if aPkg == nil {
for _, v := range aPkgMap {
aPkg = v
break
}
if aPkg == nil {
err = errors.New("no apkg found?")
return
}
}
pi.Dirname = diskPath
pi.PDoc = doc.New(aPkg, pkgName, 0)
pi.IsMain = strings.Contains(pkgName, "camlistore.org/cmd/")
return
}
// NewParser creates a new Parser reference from the given options
func NewParser(opts *ParserOptions) (*Parser, error) {
var mode parser.Mode
if opts != nil && opts.Comments {
mode = parser.ParseComments
}
fset := token.NewFileSet()
p := &Parser{fset: fset}
var err error
switch {
case opts.File != "":
p.file, err = parser.ParseFile(fset, opts.File, nil, mode)
if err != nil {
return nil, err
}
case opts.Dir != "":
p.pkgs, err = parser.ParseDir(fset, opts.Dir, nil, mode)
if err != nil {
return nil, err
}
case opts.Src != nil:
p.file, err = parser.ParseFile(fset, "src.go", opts.Src, mode)
if err != nil {
return nil, err
}
default:
return nil, errors.New("file, src or dir is not specified")
}
return p, nil
}
// ParseDir will create a Build from the directory that
// was passed into the function.
func ParseDir(path string) (*Build, error) {
var fileSet token.FileSet
packages, err := parser.ParseDir(&fileSet, path, nil, parser.AllErrors)
if err != nil {
return nil, err
}
// create new build for the file set
build := NewBuild()
// iterate over all packages in the directory
for _, pkg := range packages {
// iterate over all files within the package
for name, astTree := range pkg.Files {
baseName := filepath.Base(name)
// create a comment map from file
commentMap := ast.NewCommentMap(&fileSet, astTree, astTree.Comments)
fileAST, err := ParseFileAST(baseName, astTree, commentMap)
if err != nil {
return nil, err
}
build.AddFile(baseName, fileAST)
}
}
return build, nil
}
func parsePackage() ParsedPackage {
pkgname := "http"
dirpath := runtime.GOROOT() + "/src/pkg/net/http"
// filter function to select the desired .go files
filter := func(d os.FileInfo) bool {
if isPkgFile(d) {
// Some directories contain main packages: Only accept
// files that belong to the expected package so that
// parser.ParsePackage doesn't return "multiple packages
// found" errors.
// Additionally, accept the special package name
// fakePkgName if we are looking at cmd documentation.
name := pkgName(dirpath + "/" + d.Name())
return name == pkgname
}
return false
}
// get package AST
pkgs, err := parser.ParseDir(token.NewFileSet(), dirpath, filter, parser.ParseComments)
if err != nil {
println("parse", dirpath, err.Error())
panic("fail")
}
return pkgs
}
// ConfigureTypes configures the scope type checker
func (scope *Scope) ConfigureTypes(path string, line int) error {
scope.path = path
scope.line = line
scope.fset = token.NewFileSet() // positions are relative to fset
scope.config = &types.Config{FakeImportC: true}
// Parse the file containing this very example
// but stop after processing the imports.
f, err := parser.ParseDir(scope.fset, filepath.Dir(scope.path), nil, 0)
if err != nil {
return err
}
for _, pkg := range f {
for _, file := range pkg.Files {
scope.files = append(scope.files, file)
}
}
_, errs := scope.TypeCheck()
if len(errs) > 0 {
return errs[0]
}
return nil
}
func ParseDir(src, dst string) error {
_, err := os.Stat(dst)
if err != nil {
return err
}
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, src, nil, 0)
if err != nil {
return err
}
for _, pkg := range pkgs {
for fname, f := range pkg.Files {
ParseAst(fset, f)
bytes, err := CodeBytes(fset, f)
if err != nil {
return err
}
err = ioutil.WriteFile(path.Join(dst, path.Base(fname)), bytes, 0700)
if err != nil {
return err
}
}
}
return nil
}
func (d Dir) Packages() map[string]*ast.Package {
packages, err := parser.ParseDir(token.NewFileSet(), d.Path(), nil, 0)
if err != nil {
panic(err)
}
return packages
}
func TestBump(t *testing.T) {
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, "testdata/test1", nil, parser.Mode(0))
if err != nil {
t.Fatal(err)
}
conf := Config{
MinorDelta: 1,
}
for _, pkg := range pkgs {
for _, f := range pkg.Files {
vers, err := conf.ProcessNode(fset, f)
if err != nil {
t.Errorf("got error: %s", err)
}
if _, ok := vers["version"]; !ok {
t.Errorf("should detect `version`")
}
if _, ok := vers["VERSION"]; !ok {
t.Errorf("should detect `VERSION`")
}
if vers["version"] != "1.1.0" {
t.Errorf("expected %v: got %v", "1.1.0", vers["version"])
}
if vers["VERSION"] != "2.1.0" {
t.Errorf("expected %v: got %v", "2.1.0", vers["VERSION"])
}
}
}
}
//We want to just visit Dirs in this case
func parse(path string, pkgc chan workUnit) error {
count := 0
err := filepath.Walk(path, func(fp string, fi os.FileInfo, err error) error {
if err != nil {
fmt.Println(err)
return nil
}
if !fi.IsDir() {
return nil
}
//fmt.Println("Walking ", fp)
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, fp, nil, parser.ParseComments)
//This lets us process while we wait for the file IO and parser. Should
//might provide speedup even if we run program single-threaded (OS
//threads)
// returning non-nil stops filewalking
if err != nil {
//log.Println("In walker:", err)
return nil
}
pkgc <- workUnit{pkgs, fp}
count++
if *maxDirs != -1 && count > *maxDirs {
return fmt.Errorf("maxdirs exceeded")
}
return nil
})
return err
}
func init() {
unchecked = make(map[marker]bool)
blank = make(map[marker]bool)
pkg, err := build.Import(testPackage, "", 0)
if err != nil {
panic("failed to import test package")
}
fset := token.NewFileSet()
astPkg, err := parser.ParseDir(fset, pkg.Dir, nil, parser.ParseComments)
if err != nil {
panic("failed to parse test package")
}
for _, file := range astPkg["main"].Files {
for _, comment := range file.Comments {
text := comment.Text()
pos := fset.Position(comment.Pos())
switch text {
case "UNCHECKED\n":
unchecked[marker{pos.Filename, pos.Line}] = true
case "BLANK\n":
blank[marker{pos.Filename, pos.Line}] = true
}
}
}
}
func ParseDirectory(dir_path, ext string) (map[string]*tree.Node, os.Error) {
pkgs, err := parser.ParseDir(
token.NewFileSet(),
dir_path,
func(finfo *os.FileInfo) bool {
return strings.HasSuffix(finfo.Name, ext)
},
0)
if err != nil {
return nil, err
}
if len(pkgs) == 0 {
return nil, os.NewError("No packages found.")
}
pkgasts := make(map[string]*tree.Node)
for name, node := range pkgs {
visitor := visitor.New()
walk.GoAST_Walk(
visitor,
node,
)
pkgasts[name] = visitor.AST()
}
return pkgasts, nil
}
func parserPkg(pkgRealpath, pkgpath string) error {
rep := strings.NewReplacer("/", "_", ".", "_")
commentFilename = COMMENTFL + rep.Replace(pkgpath) + ".go"
if !compareFile(pkgRealpath) {
Info(pkgRealpath + " has not changed, not reloading")
return nil
}
genInfoList = make(map[string][]ControllerComments)
fileSet := token.NewFileSet()
astPkgs, err := parser.ParseDir(fileSet, pkgRealpath, func(info os.FileInfo) bool {
name := info.Name()
return !info.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
}, parser.ParseComments)
if err != nil {
return err
}
for _, pkg := range astPkgs {
for _, fl := range pkg.Files {
for _, d := range fl.Decls {
switch specDecl := d.(type) {
case *ast.FuncDecl:
if specDecl.Recv != nil {
parserComments(specDecl.Doc, specDecl.Name.String(), fmt.Sprint(specDecl.Recv.List[0].Type.(*ast.StarExpr).X), pkgpath)
}
}
}
}
}
genRouterCode()
savetoFile(pkgRealpath)
return nil
}
// lookInDirectory looks in the package (if any) in the directory for the named exported identifier.
func lookInDirectory(directory, name string) {
fset := token.NewFileSet()
pkgs, _ := parser.ParseDir(fset, directory, nil, parser.ParseComments) // Ignore the error.
for _, pkg := range pkgs {
doPackage(pkg, fset, name)
}
}
// Imports returns list of packages imported by
// all sources found in dir.
func Imports(dir string) ([]string, error) {
fset := token.NewFileSet()
// Find all packages in current dir
pkgs, err := parser.ParseDir(fset, dir, nil, 0)
if err != nil {
return nil, err
}
// Iterate over each package, each file
// and add imported packages to map
imports := make(map[string]struct{})
for _, pkg := range pkgs {
for _, file := range pkg.Files {
for _, impt := range file.Imports {
path := strings.Trim(impt.Path.Value, `"`)
imports[path] = struct{}{}
}
}
}
// Convert map to slice and sort
var ret []string
for name := range imports {
ret = append(ret, name)
}
sort.Strings(ret)
return ret, nil
}
func TestSorter(t *testing.T) {
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, "test", nil, parser.ParseComments)
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 1 {
t.Fatalf("Expected len(pkgs) == 1, got %d", len(pkgs))
}
expected := []string{"test", "doc", "a", "z"}
sorter := newFileSorter(pkgs["test"])
if len(sorter.names) != len(expected) {
t.Fatalf("Expected %#v, got %#v", expected, sorter.names)
}
for i := 0; i < len(sorter.names); i++ {
if sorter.names[i] != expected[i] {
t.Fatalf("Expected %s, got %s", expected, sorter.names)
}
}
}
