func getGolangDoc(importPath, etag string) (*Package, error) {
match := map[string]string{
"cred": setting.GitHubCredentials,
}
// Check revision.
commit, err := getGithubRevision("github.com/golang/go", "master")
if err != nil {
return nil, fmt.Errorf("get revision: %v", err)
}
if commit == etag {
return nil, ErrPackageNotModified
}
// Get files.
var tree struct {
Tree []struct {
Url string
Path string
Type string
}
Url string
}
if err := com.HttpGetJSON(Client,
com.Expand("https://api.github.com/repos/golang/go/git/trees/master?recursive=1&{cred}", match), &tree); err != nil {
return nil, fmt.Errorf("get tree: %v", err)
}
dirPrefix := "src/" + importPath + "/"
dirLevel := len(strings.Split(dirPrefix, "/"))
dirLength := len(dirPrefix)
dirMap := make(map[string]bool)
files := make([]com.RawFile, 0, 10)
for _, node := range tree.Tree {
// Skip directories and files in irrelevant directories.
if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
continue
}
// Get files and check if directories have acceptable files.
if d, f := path.Split(node.Path); base.IsDocFile(f) {
// Check if file is in the directory that is corresponding to import path.
if d == dirPrefix {
files = append(files, &Source{
SrcName: f,
BrowseUrl: com.Expand("github.com/golang/go/blob/master/{0}", nil, node.Path),
RawSrcUrl: com.Expand("https://raw.github.com/golang/go/master/{0}?{1}", nil, node.Path, setting.GitHubCredentials),
})
continue
}
// Otherwise, check if it's a direct sub-directory of import path.
if len(strings.Split(d, "/"))-dirLevel == 1 {
dirMap[d[dirLength:len(d)-1]] = true
continue
}
}
}
dirs := base.MapToSortedStrings(dirMap)
if len(files) == 0 && len(dirs) == 0 {
return nil, ErrPackageNoGoFile
} else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil {
return nil, fmt.Errorf("fetch files: %v", err)
}
// Start generating data.
w := &Walker{
LineFmt: "#L%d",
Pdoc: &Package{
PkgInfo: &models.PkgInfo{
ImportPath: importPath,
ProjectPath: "github.com/golang/go",
ViewDirPath: "github.com/golang/go/tree/master/src/" + importPath,
Etag: commit,
IsGoRepo: true,
Subdirs: strings.Join(dirs, "|"),
},
},
}
srcs := make([]*Source, 0, len(files))
srcMap := make(map[string]*Source)
for _, f := range files {
s := f.(*Source)
srcs = append(srcs, s)
if !strings.HasSuffix(f.Name(), "_test.go") {
srcMap[f.Name()] = s
}
}
pdoc, err := w.Build(&WalkRes{
WalkDepth: WD_All,
WalkType: WT_Memory,
WalkMode: WM_All,
Srcs: srcs,
})
if err != nil {
return nil, fmt.Errorf("walk package: %v", err)
}
return pdoc, nil
}
// Only support .zip.
func getRepoByArchive(match map[string]string, downloadPath string) (bool, string, []com.RawFile, []string, error) {
stdout, _, err := com.ExecCmd("curl", downloadPath)
if err != nil {
return false, "", nil, nil, err
}
p := []byte(stdout)
r, err := zip.NewReader(bytes.NewReader(p), int64(len(p)))
if err != nil {
return false, "", nil, nil, errors.New(downloadPath + " -> new zip: " + err.Error())
}
if len(r.File) == 0 {
return false, "", nil, nil, nil
}
nameLen := strings.Index(r.File[0].Name, "/")
dirPrefix := match["dir"]
if len(dirPrefix) != 0 {
dirPrefix = dirPrefix[1:] + "/"
}
preLen := len(dirPrefix)
isGoPro := false
// for k, v := range match {
// println(k, v)
// }
comment := r.Comment
files := make([]com.RawFile, 0, 5)
dirs := make([]string, 0, 5)
for _, f := range r.File {
fileName := f.Name[nameLen+1:]
// Skip directories and files in wrong directories, get them later.
if strings.HasSuffix(fileName, "/") || !strings.HasPrefix(fileName, dirPrefix) {
continue
}
//fmt.Println(fileName)
// Get files and check if directories have acceptable files.
if d, fn := path.Split(fileName); base.IsDocFile(fn) &&
base.FilterDirName(d) {
// Check if it's a Go file.
if !isGoPro && strings.HasSuffix(fn, ".go") {
isGoPro = true
}
// Check if file is in the directory that is corresponding to import path.
if d == dirPrefix {
// Yes.
if !isGoPro && strings.HasSuffix(fn, ".go") {
isGoPro = true
}
// Get file from archive.
rc, err := f.Open()
if err != nil {
return isGoPro, comment, files, dirs,
errors.New(downloadPath + " -> open file: " + err.Error())
}
p := make([]byte, f.FileInfo().Size())
rc.Read(p)
if err != nil {
return isGoPro, comment, files, dirs,
errors.New(downloadPath + " -> read file: " + err.Error())
}
//fmt.Println(com.Expand(match["browserUrlTpl"], match, fn))
files = append(files, &Source{
SrcName: fn,
BrowseUrl: com.Expand(match["browserUrlTpl"], match, fn),
RawSrcUrl: com.Expand(match["rawSrcUrlTpl"], match, fileName[preLen:]),
SrcData: p,
})
} else {
sd, _ := path.Split(d[preLen:])
sd = strings.TrimSuffix(sd, "/")
if !checkDir(sd, dirs) {
dirs = append(dirs, sd)
}
}
}
}
return isGoPro, comment, files, dirs, nil
}
func getGithubDoc(match map[string]string, etag string) (*Package, error) {
match["cred"] = setting.GitHubCredentials
if len(match["tag"]) == 0 {
repoInfo := new(RepoInfo)
if err := com.HttpGetJSON(Client, com.Expand("https://api.github.com/repos/{owner}/{repo}?{cred}", match), repoInfo); err != nil {
return nil, fmt.Errorf("get repo default branch: %v", err)
}
match["tag"] = repoInfo.DefaultBranch
}
// Check revision.
commit, err := getGithubRevision(com.Expand("github.com/{owner}/{repo}", match), match["tag"])
if err != nil {
return nil, fmt.Errorf("get revision: %v", err)
}
if commit == etag {
return nil, ErrPackageNotModified
}
// Get files.
var tree struct {
Tree []struct {
Url string
Path string
Type string
}
Url string
}
if err := com.HttpGetJSON(Client,
com.Expand("https://api.github.com/repos/{owner}/{repo}/git/trees/{tag}?recursive=1&{cred}", match), &tree); err != nil {
return nil, fmt.Errorf("get tree: %v", err)
}
// Because Github API URLs are case-insensitive, we need to check that the
// userRepo returned from Github matches the one that we are requesting.
if !strings.HasPrefix(tree.Url, com.Expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
return nil, errors.New("GitHub import path has incorrect case")
}
// Get source file data and subdirectories.
dirPrefix := match["dir"]
if dirPrefix != "" {
dirPrefix = dirPrefix[1:] + "/"
}
dirLevel := len(strings.Split(dirPrefix, "/"))
dirLength := len(dirPrefix)
dirMap := make(map[string]bool)
files := make([]com.RawFile, 0, 10)
for _, node := range tree.Tree {
// Skip directories and files in wrong directories, get them later.
if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
continue
}
// Get files and check if directories have acceptable files.
if d, f := path.Split(node.Path); base.IsDocFile(f) {
// Check if file is in the directory that is corresponding to import path.
if d == dirPrefix {
files = append(files, &Source{
SrcName: f,
BrowseUrl: com.Expand("github.com/{owner}/{repo}/blob/{tag}/{0}", match, node.Path),
RawSrcUrl: com.Expand("https://raw.github.com/{owner}/{repo}/{tag}/{0}?{1}", match, node.Path, setting.GitHubCredentials),
})
continue
}
// Otherwise, check if it's a direct sub-directory of import path.
if len(strings.Split(d, "/"))-dirLevel == 1 {
dirMap[d[dirLength:len(d)-1]] = true
continue
}
}
}
dirs := base.MapToSortedStrings(dirMap)
if len(files) == 0 && len(dirs) == 0 {
return nil, ErrPackageNoGoFile
} else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil {
return nil, fmt.Errorf("fetch files: %v", err)
}
// Start generating data.
// IsGoSubrepo check has been placed to crawl.getDynamic.
w := &Walker{
LineFmt: "#L%d",
Pdoc: &Package{
PkgInfo: &models.PkgInfo{
ImportPath: match["importPath"],
ProjectPath: com.Expand("github.com/{owner}/{repo}", match),
ViewDirPath: com.Expand("github.com/{owner}/{repo}/tree/{tag}/{importPath}", match),
Etag: commit,
Subdirs: strings.Join(dirs, "|"),
},
},
}
srcs := make([]*Source, 0, len(files))
srcMap := make(map[string]*Source)
for _, f := range files {
s, _ := f.(*Source)
srcs = append(srcs, s)
if !strings.HasSuffix(f.Name(), "_test.go") {
srcMap[f.Name()] = s
}
}
pdoc, err := w.Build(&WalkRes{
WalkDepth: WD_All,
WalkType: WT_Memory,
WalkMode: WM_All,
Srcs: srcs,
})
if err != nil {
return nil, fmt.Errorf("error walking package: %v", err)
}
return pdoc, nil
}
func getVCSDoc(match map[string]string, etagSaved string) (*Package, error) {
if strings.HasPrefix(match["importPath"], "golang.org/x/") {
match["owner"] = "golang"
match["repo"] = path.Dir(strings.TrimPrefix(match["importPath"], "golang.org/x/"))
return getGithubDoc(match, etagSaved)
} else if strings.HasPrefix(match["importPath"], "gopkg.in/") {
m := gopkgPathPattern.FindStringSubmatch(strings.TrimPrefix(match["importPath"], "gopkg.in"))
if m == nil {
return nil, fmt.Errorf("unsupported gopkg.in import path: %s", match["importPath"])
}
user := m[1]
repo := m[2]
if len(user) == 0 {
user = "******" + repo
}
match["owner"] = user
match["repo"] = repo
match["tag"] = m[3]
return getGithubDoc(match, etagSaved)
}
cmd := vcsCmds[match["vcs"]]
if cmd == nil {
return nil, com.NotFoundError{com.Expand("VCS not supported: {vcs}", match)}
}
scheme := match["scheme"]
if scheme == "" {
i := strings.Index(etagSaved, "-")
if i > 0 {
scheme = etagSaved[:i]
}
}
schemes := cmd.schemes
if scheme != "" {
for i := range cmd.schemes {
if cmd.schemes[i] == scheme {
schemes = cmd.schemes[i : i+1]
break
}
}
}
// Download and checkout.
tag, _, err := cmd.download(schemes, match["repo"], etagSaved)
if err != nil {
return nil, err
}
// Find source location.
urlTemplate, urlMatch, lineFmt := lookupURLTemplate(match["repo"], match["dir"], tag)
// Slurp source files.
d := path.Join(repoRoot, com.Expand("{repo}.{vcs}", match), match["dir"])
f, err := os.Open(d)
if err != nil {
if os.IsNotExist(err) {
err = com.NotFoundError{err.Error()}
}
return nil, err
}
fis, err := f.Readdir(-1)
if err != nil {
return nil, err
}
// Get source file data.
var files []com.RawFile
for _, fi := range fis {
if fi.IsDir() || !base.IsDocFile(fi.Name()) {
continue
}
b, err := ioutil.ReadFile(path.Join(d, fi.Name()))
if err != nil {
return nil, err
}
files = append(files, &Source{
SrcName: fi.Name(),
BrowseUrl: com.Expand(urlTemplate, urlMatch, fi.Name()),
SrcData: b,
})
}
// Start generating data.
w := &Walker{
LineFmt: lineFmt,
Pdoc: &Package{
PkgInfo: &models.PkgInfo{
ImportPath: match["importPath"],
},
},
}
srcs := make([]*Source, 0, len(files))
for _, f := range files {
s, _ := f.(*Source)
srcs = append(srcs, s)
}
return w.Build(&WalkRes{
WalkDepth: WD_All,
WalkType: WT_Memory,
WalkMode: WM_All,
Srcs: srcs,
})
}
