Beispiel #1
0
func getGolangDoc(importPath, etag string) (*Package, error) {
	match := map[string]string{
		"cred": setting.GitHubCredentials,
	}

	// Check revision.
	commit, err := getGithubRevision("github.com/golang/go", "master")
	if err != nil {
		return nil, fmt.Errorf("get revision: %v", err)
	}
	if commit == etag {
		return nil, ErrPackageNotModified
	}

	// Get files.
	var tree struct {
		Tree []struct {
			Url  string
			Path string
			Type string
		}
		Url string
	}

	if err := com.HttpGetJSON(Client,
		com.Expand("https://api.github.com/repos/golang/go/git/trees/master?recursive=1&{cred}", match), &tree); err != nil {
		return nil, fmt.Errorf("get tree: %v", err)
	}

	dirPrefix := "src/" + importPath + "/"
	dirLevel := len(strings.Split(dirPrefix, "/"))
	dirLength := len(dirPrefix)
	dirMap := make(map[string]bool)
	files := make([]com.RawFile, 0, 10)

	for _, node := range tree.Tree {
		// Skip directories and files in irrelevant directories.
		if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
			continue
		}

		// Get files and check if directories have acceptable files.
		if d, f := path.Split(node.Path); base.IsDocFile(f) {
			// Check if file is in the directory that is corresponding to import path.
			if d == dirPrefix {
				files = append(files, &Source{
					SrcName:   f,
					BrowseUrl: com.Expand("github.com/golang/go/blob/master/{0}", nil, node.Path),
					RawSrcUrl: com.Expand("https://raw.github.com/golang/go/master/{0}?{1}", nil, node.Path, setting.GitHubCredentials),
				})
				continue
			}

			// Otherwise, check if it's a direct sub-directory of import path.
			if len(strings.Split(d, "/"))-dirLevel == 1 {
				dirMap[d[dirLength:len(d)-1]] = true
				continue
			}
		}
	}

	dirs := base.MapToSortedStrings(dirMap)

	if len(files) == 0 && len(dirs) == 0 {
		return nil, ErrPackageNoGoFile
	} else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil {
		return nil, fmt.Errorf("fetch files: %v", err)
	}

	// Start generating data.
	w := &Walker{
		LineFmt: "#L%d",
		Pdoc: &Package{
			PkgInfo: &models.PkgInfo{
				ImportPath:  importPath,
				ProjectPath: "github.com/golang/go",
				ViewDirPath: "github.com/golang/go/tree/master/src/" + importPath,
				Etag:        commit,
				IsGoRepo:    true,
				Subdirs:     strings.Join(dirs, "|"),
			},
		},
	}

	srcs := make([]*Source, 0, len(files))
	srcMap := make(map[string]*Source)
	for _, f := range files {
		s := f.(*Source)
		srcs = append(srcs, s)

		if !strings.HasSuffix(f.Name(), "_test.go") {
			srcMap[f.Name()] = s
		}
	}

	pdoc, err := w.Build(&WalkRes{
		WalkDepth: WD_All,
		WalkType:  WT_Memory,
		WalkMode:  WM_All,
		Srcs:      srcs,
	})
	if err != nil {
		return nil, fmt.Errorf("walk package: %v", err)
	}

	return pdoc, nil
}
Beispiel #2
0
// Only support .zip.
func getRepoByArchive(match map[string]string, downloadPath string) (bool, string, []com.RawFile, []string, error) {
	stdout, _, err := com.ExecCmd("curl", downloadPath)
	if err != nil {
		return false, "", nil, nil, err
	}
	p := []byte(stdout)

	r, err := zip.NewReader(bytes.NewReader(p), int64(len(p)))
	if err != nil {
		return false, "", nil, nil, errors.New(downloadPath + " -> new zip: " + err.Error())
	}

	if len(r.File) == 0 {
		return false, "", nil, nil, nil
	}

	nameLen := strings.Index(r.File[0].Name, "/")
	dirPrefix := match["dir"]
	if len(dirPrefix) != 0 {
		dirPrefix = dirPrefix[1:] + "/"
	}
	preLen := len(dirPrefix)
	isGoPro := false

	// for k, v := range match {
	// 	println(k, v)
	// }
	comment := r.Comment

	files := make([]com.RawFile, 0, 5)
	dirs := make([]string, 0, 5)
	for _, f := range r.File {
		fileName := f.Name[nameLen+1:]
		// Skip directories and files in wrong directories, get them later.
		if strings.HasSuffix(fileName, "/") || !strings.HasPrefix(fileName, dirPrefix) {
			continue
		}
		//fmt.Println(fileName)

		// Get files and check if directories have acceptable files.
		if d, fn := path.Split(fileName); base.IsDocFile(fn) &&
			base.FilterDirName(d) {
			// Check if it's a Go file.
			if !isGoPro && strings.HasSuffix(fn, ".go") {
				isGoPro = true
			}

			// Check if file is in the directory that is corresponding to import path.
			if d == dirPrefix {
				// Yes.
				if !isGoPro && strings.HasSuffix(fn, ".go") {
					isGoPro = true
				}
				// Get file from archive.
				rc, err := f.Open()
				if err != nil {
					return isGoPro, comment, files, dirs,
						errors.New(downloadPath + " -> open file: " + err.Error())
				}

				p := make([]byte, f.FileInfo().Size())
				rc.Read(p)
				if err != nil {
					return isGoPro, comment, files, dirs,
						errors.New(downloadPath + " -> read file: " + err.Error())
				}
				//fmt.Println(com.Expand(match["browserUrlTpl"], match, fn))
				files = append(files, &Source{
					SrcName:   fn,
					BrowseUrl: com.Expand(match["browserUrlTpl"], match, fn),
					RawSrcUrl: com.Expand(match["rawSrcUrlTpl"], match, fileName[preLen:]),
					SrcData:   p,
				})

			} else {
				sd, _ := path.Split(d[preLen:])
				sd = strings.TrimSuffix(sd, "/")
				if !checkDir(sd, dirs) {
					dirs = append(dirs, sd)
				}
			}
		}
	}
	return isGoPro, comment, files, dirs, nil
}
Beispiel #3
0
func getGithubDoc(match map[string]string, etag string) (*Package, error) {
	match["cred"] = setting.GitHubCredentials
	if len(match["tag"]) == 0 {
		repoInfo := new(RepoInfo)
		if err := com.HttpGetJSON(Client, com.Expand("https://api.github.com/repos/{owner}/{repo}?{cred}", match), repoInfo); err != nil {
			return nil, fmt.Errorf("get repo default branch: %v", err)
		}

		match["tag"] = repoInfo.DefaultBranch
	}

	// Check revision.
	commit, err := getGithubRevision(com.Expand("github.com/{owner}/{repo}", match), match["tag"])
	if err != nil {
		return nil, fmt.Errorf("get revision: %v", err)
	}
	if commit == etag {
		return nil, ErrPackageNotModified
	}

	// Get files.
	var tree struct {
		Tree []struct {
			Url  string
			Path string
			Type string
		}
		Url string
	}

	if err := com.HttpGetJSON(Client,
		com.Expand("https://api.github.com/repos/{owner}/{repo}/git/trees/{tag}?recursive=1&{cred}", match), &tree); err != nil {
		return nil, fmt.Errorf("get tree: %v", err)
	}

	// Because Github API URLs are case-insensitive, we need to check that the
	// userRepo returned from Github matches the one that we are requesting.
	if !strings.HasPrefix(tree.Url, com.Expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
		return nil, errors.New("GitHub import path has incorrect case")
	}

	// Get source file data and subdirectories.
	dirPrefix := match["dir"]
	if dirPrefix != "" {
		dirPrefix = dirPrefix[1:] + "/"
	}
	dirLevel := len(strings.Split(dirPrefix, "/"))
	dirLength := len(dirPrefix)
	dirMap := make(map[string]bool)
	files := make([]com.RawFile, 0, 10)

	for _, node := range tree.Tree {
		// Skip directories and files in wrong directories, get them later.
		if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
			continue
		}

		// Get files and check if directories have acceptable files.
		if d, f := path.Split(node.Path); base.IsDocFile(f) {
			// Check if file is in the directory that is corresponding to import path.
			if d == dirPrefix {
				files = append(files, &Source{
					SrcName:   f,
					BrowseUrl: com.Expand("github.com/{owner}/{repo}/blob/{tag}/{0}", match, node.Path),
					RawSrcUrl: com.Expand("https://raw.github.com/{owner}/{repo}/{tag}/{0}?{1}", match, node.Path, setting.GitHubCredentials),
				})
				continue
			}

			// Otherwise, check if it's a direct sub-directory of import path.
			if len(strings.Split(d, "/"))-dirLevel == 1 {
				dirMap[d[dirLength:len(d)-1]] = true
				continue
			}
		}
	}

	dirs := base.MapToSortedStrings(dirMap)

	if len(files) == 0 && len(dirs) == 0 {
		return nil, ErrPackageNoGoFile
	} else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil {
		return nil, fmt.Errorf("fetch files: %v", err)
	}

	// Start generating data.
	// IsGoSubrepo check has been placed to crawl.getDynamic.
	w := &Walker{
		LineFmt: "#L%d",
		Pdoc: &Package{
			PkgInfo: &models.PkgInfo{
				ImportPath:  match["importPath"],
				ProjectPath: com.Expand("github.com/{owner}/{repo}", match),
				ViewDirPath: com.Expand("github.com/{owner}/{repo}/tree/{tag}/{importPath}", match),
				Etag:        commit,
				Subdirs:     strings.Join(dirs, "|"),
			},
		},
	}

	srcs := make([]*Source, 0, len(files))
	srcMap := make(map[string]*Source)
	for _, f := range files {
		s, _ := f.(*Source)
		srcs = append(srcs, s)

		if !strings.HasSuffix(f.Name(), "_test.go") {
			srcMap[f.Name()] = s
		}
	}

	pdoc, err := w.Build(&WalkRes{
		WalkDepth: WD_All,
		WalkType:  WT_Memory,
		WalkMode:  WM_All,
		Srcs:      srcs,
	})
	if err != nil {
		return nil, fmt.Errorf("error walking package: %v", err)
	}

	return pdoc, nil
}
Beispiel #4
0
func getVCSDoc(match map[string]string, etagSaved string) (*Package, error) {
	if strings.HasPrefix(match["importPath"], "golang.org/x/") {
		match["owner"] = "golang"
		match["repo"] = path.Dir(strings.TrimPrefix(match["importPath"], "golang.org/x/"))
		return getGithubDoc(match, etagSaved)
	} else if strings.HasPrefix(match["importPath"], "gopkg.in/") {
		m := gopkgPathPattern.FindStringSubmatch(strings.TrimPrefix(match["importPath"], "gopkg.in"))
		if m == nil {
			return nil, fmt.Errorf("unsupported gopkg.in import path: %s", match["importPath"])
		}
		user := m[1]
		repo := m[2]
		if len(user) == 0 {
			user = "******" + repo
		}
		match["owner"] = user
		match["repo"] = repo
		match["tag"] = m[3]
		return getGithubDoc(match, etagSaved)
	}

	cmd := vcsCmds[match["vcs"]]
	if cmd == nil {
		return nil, com.NotFoundError{com.Expand("VCS not supported: {vcs}", match)}
	}

	scheme := match["scheme"]
	if scheme == "" {
		i := strings.Index(etagSaved, "-")
		if i > 0 {
			scheme = etagSaved[:i]
		}
	}

	schemes := cmd.schemes
	if scheme != "" {
		for i := range cmd.schemes {
			if cmd.schemes[i] == scheme {
				schemes = cmd.schemes[i : i+1]
				break
			}
		}
	}

	// Download and checkout.

	tag, _, err := cmd.download(schemes, match["repo"], etagSaved)
	if err != nil {
		return nil, err
	}

	// Find source location.

	urlTemplate, urlMatch, lineFmt := lookupURLTemplate(match["repo"], match["dir"], tag)

	// Slurp source files.

	d := path.Join(repoRoot, com.Expand("{repo}.{vcs}", match), match["dir"])
	f, err := os.Open(d)
	if err != nil {
		if os.IsNotExist(err) {
			err = com.NotFoundError{err.Error()}
		}
		return nil, err
	}
	fis, err := f.Readdir(-1)
	if err != nil {
		return nil, err
	}

	// Get source file data.
	var files []com.RawFile
	for _, fi := range fis {
		if fi.IsDir() || !base.IsDocFile(fi.Name()) {
			continue
		}
		b, err := ioutil.ReadFile(path.Join(d, fi.Name()))
		if err != nil {
			return nil, err
		}
		files = append(files, &Source{
			SrcName:   fi.Name(),
			BrowseUrl: com.Expand(urlTemplate, urlMatch, fi.Name()),
			SrcData:   b,
		})
	}

	// Start generating data.
	w := &Walker{
		LineFmt: lineFmt,
		Pdoc: &Package{
			PkgInfo: &models.PkgInfo{
				ImportPath: match["importPath"],
			},
		},
	}

	srcs := make([]*Source, 0, len(files))
	for _, f := range files {
		s, _ := f.(*Source)
		srcs = append(srcs, s)
	}

	return w.Build(&WalkRes{
		WalkDepth: WD_All,
		WalkType:  WT_Memory,
		WalkMode:  WM_All,
		Srcs:      srcs,
	})
}