func main() {
log.Println("201402191")
client := &http.Client{}
p, err := com.HttpGetBytes(client, "http://godoc.org/-/index", nil)
if err != nil {
log.Fatalf("Fail to load page: %v", err)
}
content := string(p)
start := strings.Index(content, "<tbody>") + 7
end := strings.Index(content, "</tbody>")
content = content[start:end]
pkgs := strings.Split(content, "<tr>")[1:]
skipUntilIndex := 9052
endWhenIndex := 12000
for i, name := range pkgs {
if i < skipUntilIndex {
continue
} else if i == endWhenIndex {
break
}
name = strings.TrimSpace(name)[14:]
end := strings.Index(name, "\">")
name = name[:end]
log.Printf("#%d %s", i, name)
_, err = com.HttpGet(client, "https://gowalker.org/"+name, nil)
if err != nil {
log.Fatalf("Fail to load page: %v", err)
}
time.Sleep(0 * time.Second)
}
}
func ExampleHttpGet() ([]byte, error) {
rc, err := com.HttpGet(&http.Client{}, "http://gowalker.org", nil)
if err != nil {
return nil, err
}
p, err := ioutil.ReadAll(rc)
rc.Close()
return p, err
}
// getLaunchpadDoc downloads tarball from launchpad.net.
func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) {
if match["project"] != "" && match["series"] != "" {
rc, err := com.HttpGet(client, com.Expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
_, isNotFound := err.(com.NotFoundError)
switch {
case err == nil:
rc.Close()
// The structure of the import path is launchpad.net/{root}/{dir}.
case isNotFound:
// The structure of the import path is is launchpad.net/{project}/{dir}.
match["repo"] = match["project"]
match["dir"] = com.Expand("{series}{dir}", match)
default:
return nil, err
}
}
var downloadPath string
// Check if download with specific revision.
if len(nod.Value) == 0 {
downloadPath = com.Expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match)
} else {
downloadPath = com.Expand("https://bazaar.launchpad.net/+branch/{repo}/tarball/"+nod.Value, match)
}
// Scrape the repo browser to find the project revision and individual Go files.
p, err := com.HttpGetBytes(client, downloadPath, nil)
if err != nil {
return nil, err
}
installPath := installRepoPath + "/" + nod.ImportPath
// Remove old files.
os.RemoveAll(installPath + "/")
os.MkdirAll(installPath+"/", os.ModePerm)
gzr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return nil, err
}
defer gzr.Close()
tr := tar.NewReader(gzr)
var autoPath string // Auto path is the root path that generated by bitbucket.org.
// Get source file data.
dirs := make([]string, 0, 5)
for {
h, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
fn := h.Name
// Check root path.
if len(autoPath) == 0 {
autoPath = fn[:strings.Index(fn, match["repo"])+len(match["repo"])]
}
absPath := strings.Replace(fn, autoPath, installPath, 1)
switch {
case h.FileInfo().IsDir(): // Directory.
// Create diretory before create file.
os.MkdirAll(absPath+"/", os.ModePerm)
// Check if current directory is example.
if !(!ctx.Bool("example") && strings.Contains(absPath, "example")) {
dirs = append(dirs, absPath)
}
case !strings.HasPrefix(fn, "."):
// Get data from archive.
fbytes := make([]byte, h.Size)
if _, err := io.ReadFull(tr, fbytes); err != nil {
return nil, err
}
if err = com.WriteFile(absPath, fbytes); err != nil {
return nil, err
}
}
}
var imports []string
// Check if need to check imports.
if nod.IsGetDeps {
for _, d := range dirs {
importPkgs, err := CheckImports(d+"/", match["importPath"], nod)
if err != nil {
return nil, err
}
imports = append(imports, importPkgs...)
}
}
return imports, err
}
func getLaunchpadDoc(client *http.Client, match map[string]string, tag, savedEtag string) (*hv.Package, error) {
if match["project"] != "" && match["series"] != "" {
rc, err := com.HttpGet(client, com.Expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
switch {
case err == nil:
rc.Close()
// The structure of the import path is launchpad.net/{root}/{dir}.
case isNotFound(err):
// The structure of the import path is is launchpad.net/{project}/{dir}.
match["repo"] = match["project"]
match["dir"] = com.Expand("{series}{dir}", match)
default:
return nil, err
}
}
// Scrape the repo browser to find the project revision and individual Go files.
p, err := com.HttpGetBytes(client, com.Expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match), nil)
if err != nil {
return nil, err
}
// Get source file data.
gzr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return nil, err
}
defer gzr.Close()
tr := tar.NewReader(gzr)
var hash []byte
dirPrefix := com.Expand("+branch/{repo}{dir}/", match)
preLen := len(dirPrefix)
isGoPro := false // Indicates whether it's a Go project.
isRootPath := match["importPath"] == utils.GetProjectPath(match["importPath"])
dirs := make([]string, 0, 3)
files := make([]com.RawFile, 0, 5)
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
// Skip directories and files in wrong directories, get them later.
if strings.HasSuffix(h.Name, "/") || !strings.HasPrefix(h.Name, dirPrefix) {
continue
}
d, f := path.Split(h.Name)
if utils.IsDocFile(f) && utils.FilterDirName(d) {
// Check if it's a Go file.
if isRootPath && !isGoPro && strings.HasSuffix(f, ".go") {
isGoPro = true
}
// Get file from archive.
b := make([]byte, h.Size)
if _, err := io.ReadFull(tr, b); err != nil {
return nil, err
}
m := md5.New()
m.Write(b)
hash = m.Sum(hash)
// Check if file is in the directory that is corresponding to import path.
if d == dirPrefix {
// Yes.
if !isRootPath && !isGoPro && strings.HasSuffix(f, ".go") {
isGoPro = true
}
files = append(files, &hv.Source{
SrcName: f,
BrowseUrl: com.Expand("bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/{0}", match, f),
SrcData: b})
} else {
sd, _ := path.Split(d[preLen:])
sd = strings.TrimSuffix(sd, "/")
if !checkDir(sd, dirs) {
dirs = append(dirs, sd)
}
}
}
}
if !isGoPro {
return nil, com.NotFoundError{"Cannot find Go files, it's not a Go project."}
}
if len(files) == 0 && len(dirs) == 0 {
return nil, com.NotFoundError{"Directory tree does not contain Go files and subdirs."}
}
sort.Sort(byHash(hash))
m := md5.New()
m.Write(hash)
hash = m.Sum(hash[:0])
etag := hex.EncodeToString(hash)
if etag == savedEtag {
return nil, errNotModified
}
// Start generating data.
w := &hv.Walker{
LineFmt: "#L%d",
Pdoc: &hv.Package{
PkgInfo: &hv.PkgInfo{
ImportPath: match["importPath"],
ProjectName: match["repo"],
ProjectPath: com.Expand("bazaar.launchpad.net/+branch/{repo}/files", match),
ViewDirPath: com.Expand("bazaar.launchpad.net/+branch/{repo}/files/head:{dir}/", match),
Ptag: etag,
Vcs: "Launchpad",
},
PkgDecl: &hv.PkgDecl{
Dirs: dirs,
},
},
}
srcs := make([]*hv.Source, 0, len(files))
srcMap := make(map[string]*hv.Source)
for _, f := range files {
s, _ := f.(*hv.Source)
srcs = append(srcs, s)
if !strings.HasSuffix(f.Name(), "_test.go") {
srcMap[f.Name()] = s
}
}
pdoc, err := w.Build(&hv.WalkRes{
WalkDepth: hv.WD_All,
WalkType: hv.WT_Memory,
WalkMode: hv.WM_All,
Srcs: srcs,
})
if err != nil {
return nil, errors.New("doc.getLaunchpadDoc(" + match["importPath"] + ") -> Fail to build: " + err.Error())
}
if len(tag) == 0 && w.Pdoc.IsCmd {
err = generateHv(match["importPath"], srcMap)
}
return pdoc, err
}