Beispiel #1
0
func main() {
	log.Println("201402191")
	client := &http.Client{}
	p, err := com.HttpGetBytes(client, "http://godoc.org/-/index", nil)
	if err != nil {
		log.Fatalf("Fail to load page: %v", err)
	}
	content := string(p)
	start := strings.Index(content, "<tbody>") + 7
	end := strings.Index(content, "</tbody>")
	content = content[start:end]

	pkgs := strings.Split(content, "<tr>")[1:]

	skipUntilIndex := 9052
	endWhenIndex := 12000
	for i, name := range pkgs {
		if i < skipUntilIndex {
			continue
		} else if i == endWhenIndex {
			break
		}
		name = strings.TrimSpace(name)[14:]
		end := strings.Index(name, "\">")
		name = name[:end]
		log.Printf("#%d %s", i, name)
		_, err = com.HttpGet(client, "https://gowalker.org/"+name, nil)
		if err != nil {
			log.Fatalf("Fail to load page: %v", err)
		}
		time.Sleep(0 * time.Second)
	}
}
Beispiel #2
0
func ExampleHttpGet() ([]byte, error) {
	rc, err := com.HttpGet(&http.Client{}, "http://gowalker.org", nil)
	if err != nil {
		return nil, err
	}
	p, err := ioutil.ReadAll(rc)
	rc.Close()
	return p, err
}
Beispiel #3
0
// getLaunchpadDoc downloads tarball from launchpad.net.
func getLaunchpadDoc(client *http.Client, match map[string]string, installRepoPath string, nod *Node, ctx *cli.Context) ([]string, error) {

	if match["project"] != "" && match["series"] != "" {
		rc, err := com.HttpGet(client, com.Expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
		_, isNotFound := err.(com.NotFoundError)
		switch {
		case err == nil:
			rc.Close()
			// The structure of the import path is launchpad.net/{root}/{dir}.
		case isNotFound:
			// The structure of the import path is is launchpad.net/{project}/{dir}.
			match["repo"] = match["project"]
			match["dir"] = com.Expand("{series}{dir}", match)
		default:
			return nil, err
		}
	}

	var downloadPath string
	// Check if download with specific revision.
	if len(nod.Value) == 0 {
		downloadPath = com.Expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match)
	} else {
		downloadPath = com.Expand("https://bazaar.launchpad.net/+branch/{repo}/tarball/"+nod.Value, match)
	}

	// Scrape the repo browser to find the project revision and individual Go files.
	p, err := com.HttpGetBytes(client, downloadPath, nil)
	if err != nil {
		return nil, err
	}

	installPath := installRepoPath + "/" + nod.ImportPath

	// Remove old files.
	os.RemoveAll(installPath + "/")
	os.MkdirAll(installPath+"/", os.ModePerm)

	gzr, err := gzip.NewReader(bytes.NewReader(p))
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)

	var autoPath string // Auto path is the root path that generated by bitbucket.org.
	// Get source file data.
	dirs := make([]string, 0, 5)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		fn := h.Name
		// Check root path.
		if len(autoPath) == 0 {
			autoPath = fn[:strings.Index(fn, match["repo"])+len(match["repo"])]
		}
		absPath := strings.Replace(fn, autoPath, installPath, 1)

		switch {
		case h.FileInfo().IsDir(): // Directory.
			// Create diretory before create file.
			os.MkdirAll(absPath+"/", os.ModePerm)
			// Check if current directory is example.
			if !(!ctx.Bool("example") && strings.Contains(absPath, "example")) {
				dirs = append(dirs, absPath)
			}
		case !strings.HasPrefix(fn, "."):
			// Get data from archive.
			fbytes := make([]byte, h.Size)
			if _, err := io.ReadFull(tr, fbytes); err != nil {
				return nil, err
			}

			if err = com.WriteFile(absPath, fbytes); err != nil {
				return nil, err
			}
		}
	}

	var imports []string

	// Check if need to check imports.
	if nod.IsGetDeps {
		for _, d := range dirs {
			importPkgs, err := CheckImports(d+"/", match["importPath"], nod)
			if err != nil {
				return nil, err
			}
			imports = append(imports, importPkgs...)
		}
	}

	return imports, err
}
Beispiel #4
0
func getLaunchpadDoc(client *http.Client, match map[string]string, tag, savedEtag string) (*hv.Package, error) {

	if match["project"] != "" && match["series"] != "" {
		rc, err := com.HttpGet(client, com.Expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
		switch {
		case err == nil:
			rc.Close()
			// The structure of the import path is launchpad.net/{root}/{dir}.
		case isNotFound(err):
			// The structure of the import path is is launchpad.net/{project}/{dir}.
			match["repo"] = match["project"]
			match["dir"] = com.Expand("{series}{dir}", match)
		default:
			return nil, err
		}
	}

	// Scrape the repo browser to find the project revision and individual Go files.
	p, err := com.HttpGetBytes(client, com.Expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match), nil)
	if err != nil {
		return nil, err
	}

	// Get source file data.
	gzr, err := gzip.NewReader(bytes.NewReader(p))
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)

	var hash []byte
	dirPrefix := com.Expand("+branch/{repo}{dir}/", match)
	preLen := len(dirPrefix)

	isGoPro := false // Indicates whether it's a Go project.
	isRootPath := match["importPath"] == utils.GetProjectPath(match["importPath"])
	dirs := make([]string, 0, 3)
	files := make([]com.RawFile, 0, 5)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}

		// Skip directories and files in wrong directories, get them later.
		if strings.HasSuffix(h.Name, "/") || !strings.HasPrefix(h.Name, dirPrefix) {
			continue
		}

		d, f := path.Split(h.Name)
		if utils.IsDocFile(f) && utils.FilterDirName(d) {
			// Check if it's a Go file.
			if isRootPath && !isGoPro && strings.HasSuffix(f, ".go") {
				isGoPro = true
			}

			// Get file from archive.
			b := make([]byte, h.Size)
			if _, err := io.ReadFull(tr, b); err != nil {
				return nil, err
			}

			m := md5.New()
			m.Write(b)
			hash = m.Sum(hash)

			// Check if file is in the directory that is corresponding to import path.
			if d == dirPrefix {
				// Yes.
				if !isRootPath && !isGoPro && strings.HasSuffix(f, ".go") {
					isGoPro = true
				}
				files = append(files, &hv.Source{
					SrcName:   f,
					BrowseUrl: com.Expand("bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/{0}", match, f),
					SrcData:   b})
			} else {
				sd, _ := path.Split(d[preLen:])
				sd = strings.TrimSuffix(sd, "/")
				if !checkDir(sd, dirs) {
					dirs = append(dirs, sd)
				}
			}
		}
	}

	if !isGoPro {
		return nil, com.NotFoundError{"Cannot find Go files, it's not a Go project."}
	}

	if len(files) == 0 && len(dirs) == 0 {
		return nil, com.NotFoundError{"Directory tree does not contain Go files and subdirs."}
	}

	sort.Sort(byHash(hash))
	m := md5.New()
	m.Write(hash)
	hash = m.Sum(hash[:0])
	etag := hex.EncodeToString(hash)
	if etag == savedEtag {
		return nil, errNotModified
	}

	// Start generating data.
	w := &hv.Walker{
		LineFmt: "#L%d",
		Pdoc: &hv.Package{
			PkgInfo: &hv.PkgInfo{
				ImportPath:  match["importPath"],
				ProjectName: match["repo"],
				ProjectPath: com.Expand("bazaar.launchpad.net/+branch/{repo}/files", match),
				ViewDirPath: com.Expand("bazaar.launchpad.net/+branch/{repo}/files/head:{dir}/", match),
				Ptag:        etag,
				Vcs:         "Launchpad",
			},
			PkgDecl: &hv.PkgDecl{
				Dirs: dirs,
			},
		},
	}

	srcs := make([]*hv.Source, 0, len(files))
	srcMap := make(map[string]*hv.Source)
	for _, f := range files {
		s, _ := f.(*hv.Source)
		srcs = append(srcs, s)

		if !strings.HasSuffix(f.Name(), "_test.go") {
			srcMap[f.Name()] = s
		}
	}

	pdoc, err := w.Build(&hv.WalkRes{
		WalkDepth: hv.WD_All,
		WalkType:  hv.WT_Memory,
		WalkMode:  hv.WM_All,
		Srcs:      srcs,
	})
	if err != nil {
		return nil, errors.New("doc.getLaunchpadDoc(" + match["importPath"] + ") -> Fail to build: " + err.Error())
	}

	if len(tag) == 0 && w.Pdoc.IsCmd {
		err = generateHv(match["importPath"], srcMap)
	}

	return pdoc, err
}