// GetLaunchpadDoc downloads tarball from launchpad.net.
func GetLaunchpadDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
if match["project"] != "" && match["series"] != "" {
rc, err := httpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
switch {
case err == nil:
rc.Close()
// The structure of the import path is launchpad.net/{root}/{dir}.
case isNotFound(err):
// The structure of the import path is is launchpad.net/{project}/{dir}.
match["repo"] = match["project"]
match["dir"] = expand("{series}{dir}", match)
default:
return nil, err
}
}
// bundle and snapshot will have commit 'B' and 'S',
// but does not need to download dependencies.
isCheckImport := len(node.Value) == 0
var downloadPath string
// Check if download with specific revision.
if isCheckImport || len(node.Value) == 1 {
downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match)
node.Type = "commit"
} else {
downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball/"+node.Value, match)
}
// Scrape the repo browser to find the project revision and individual Go files.
p, err := HttpGetBytes(client, downloadPath, nil)
if err != nil {
return nil, err
}
projectPath := expand("launchpad.net/{repo}", match)
installPath := installGOPATH + "/src/" + projectPath
node.ImportPath = projectPath
// Remove old files.
os.RemoveAll(installPath + "/")
// Create destination directory.
os.MkdirAll(installPath+"/", os.ModePerm)
gzr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return nil, err
}
defer gzr.Close()
tr := tar.NewReader(gzr)
isCodeOnly := cmdFlags["-c"]
var autoPath string // Auto path is the root path that generated by bitbucket.org.
// Get source file data.
dirs := make([]string, 0, 5)
for {
h, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
fn := h.FileInfo().Name()
// Check root path.
if len(autoPath) == 0 {
autoPath = fn[:strings.Index(fn, match["repo"])+len(match["repo"])]
}
absPath := strings.Replace(fn, autoPath, installPath, 1)
switch {
case h.FileInfo().IsDir(): // Directory.
// Check if current directory is example.
if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
dirs = append(dirs, absPath)
}
case isCodeOnly && !utils.IsDocFile(path.Base(absPath)):
continue
case !strings.HasPrefix(fn, "."):
// Create diretory before create file.
os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm)
// Get data from archive.
fbytes := make([]byte, h.Size)
if _, err := io.ReadFull(tr, fbytes); err != nil {
return nil, err
}
// Write data to file
fw, err := os.Create(absPath)
if err != nil {
return nil, err
}
_, err = fw.Write(fbytes)
fw.Close()
if err != nil {
return nil, err
}
}
}
var imports []string
// Check if need to check imports.
if isCheckImport {
for _, d := range dirs {
importPkgs, err := CheckImports(d+"/", match["importPath"])
if err != nil {
return nil, err
}
imports = append(imports, importPkgs...)
}
}
return imports, err
}
// GetGoogleDoc downloads raw files from code.google.com.
func GetGoogleDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
setupGoogleMatch(match)
// Check version control.
if m := googleEtagRe.FindStringSubmatch(node.Value); m != nil {
match["vcs"] = m[1]
} else if err := getGoogleVCS(client, match); err != nil {
return nil, err
}
// bundle and snapshot will have commit 'B' and 'S',
// but does not need to download dependencies.
isCheckImport := len(node.Value) == 0
if len(node.Value) == 1 {
node.Value = ""
}
rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match)
// Scrape the repo browser to find the project revision and individual Go files.
p, err := HttpGetBytes(client, rootPath+"?r="+node.Value, nil)
if err != nil {
return nil, err
}
// Check revision tag.
if m := googleRevisionRe.FindSubmatch(p); m == nil {
return nil,
errors.New("doc.GetGoogleDoc(): Could not find revision for " + match["importPath"])
} else {
node.Type = "commit"
node.Value = string(m[1])
}
projectPath := expand("code.google.com/p/{repo}{dot}{subrepo}{dir}", match)
installPath := installGOPATH + "/src/" + projectPath
node.ImportPath = projectPath
// Remove old files.
os.RemoveAll(installPath + "/")
// Create destination directory.
os.MkdirAll(installPath+"/", os.ModePerm)
isCodeOnly := cmdFlags["-c"]
// Get source files in root path.
files := make([]*source, 0, 5)
for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
fname := strings.Split(string(m[1]), "?")[0]
if isCodeOnly && !utils.IsDocFile(fname) {
continue
} else if strings.HasPrefix(fname, ".") {
continue
}
files = append(files, &source{
name: fname,
rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + node.Value,
})
}
// Fetch files from VCS.
if err := fetchFiles(client, files, nil); err != nil {
return nil, err
}
// Save files.
for _, f := range files {
absPath := installPath + "/"
// Create diretory before create file.
os.MkdirAll(path.Dir(absPath), os.ModePerm)
// Write data to file
fw, err := os.Create(absPath + f.name)
if err != nil {
return nil, err
}
_, err = fw.Write(f.data)
fw.Close()
if err != nil {
return nil, err
}
}
dirs := make([]string, 0, 3)
// Get subdirectories.
for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
dirName := strings.Split(string(m[1]), "?")[0]
if strings.HasSuffix(dirName, "/") {
dirs = append(dirs, dirName)
}
}
err = downloadFiles(client, match, rootPath, installPath+"/", node.Value, dirs)
if err != nil {
return nil, err
}
var imports []string
// Check if need to check imports.
if isCheckImport {
rootdir, err := os.Open(installPath + "/")
if err != nil {
return nil, err
}
defer rootdir.Close()
dirs, err := rootdir.Readdir(0)
if err != nil {
return nil, err
}
for _, d := range dirs {
if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) {
absPath := installPath + "/" + d.Name() + "/"
importPkgs, err := CheckImports(absPath, match["importPath"])
if err != nil {
return nil, err
}
imports = append(imports, importPkgs...)
}
}
}
return imports, err
}
// GetGithubDoc downloads tarball from github.com.
func GetGithubDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
match["cred"] = githubCred
// JSON struct for github.com.
var refs []*struct {
Ref string
Url string
Object struct {
Sha string
Type string
Url string
}
}
// bundle and snapshot will have commit 'B' and 'S',
// but does not need to download dependencies.
isCheckImport := len(node.Value) == 0
switch {
case isCheckImport || len(node.Value) == 1:
// Get up-to-date version.
err := httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), &refs)
if err != nil {
return nil, err
}
tags := make(map[string]string)
for _, ref := range refs {
switch {
case strings.HasPrefix(ref.Ref, "refs/heads/"):
tags[ref.Ref[len("refs/heads/"):]] = ref.Object.Sha
case strings.HasPrefix(ref.Ref, "refs/tags/"):
tags[ref.Ref[len("refs/tags/"):]] = ref.Object.Sha
}
}
// Check revision tag.
match["tag"], match["sha"], err = bestTag(tags, "master")
if err != nil {
return nil, err
}
node.Type = "commit"
node.Value = match["sha"]
case !isCheckImport: // Bundle or snapshot.
// Check downlaod type.
switch node.Type {
case "tag", "commit", "branch":
match["sha"] = node.Value
default:
return nil, errors.New("Unknown node type: " + node.Type)
}
}
// We use .zip here.
// zip : https://github.com/{owner}/{repo}/archive/{sha}.zip
// tarball : https://github.com/{owner}/{repo}/tarball/{sha}
// Downlaod archive.
p, err := HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil)
if err != nil {
return nil, err
}
shaName := expand("{repo}-{sha}", match)
if node.Type == "tag" {
shaName = strings.Replace(shaName, "-v", "-", 1)
}
projectPath := expand("github.com/{owner}/{repo}", match)
installPath := installGOPATH + "/src/" + projectPath
node.ImportPath = projectPath
// Remove old files.
os.RemoveAll(installPath + "/")
// Create destination directory.
os.MkdirAll(installPath+"/", os.ModePerm)
r, err := zip.NewReader(bytes.NewReader(p), int64(len(p)))
if err != nil {
return nil, err
}
isCodeOnly := cmdFlags["-c"]
dirs := make([]string, 0, 5)
// Need to add root path because we cannot get from tarball.
dirs = append(dirs, installPath+"/")
for _, f := range r.File {
absPath := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1)
// Create diretory before create file.
os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm)
compareDir:
switch {
case strings.HasSuffix(absPath, "/"): // Directory.
// Check if current directory is example.
if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
for _, d := range dirs {
if d == absPath {
break compareDir
}
}
dirs = append(dirs, absPath)
}
case isCodeOnly && !utils.IsDocFile(path.Base(absPath)):
continue
case !strings.HasPrefix(f.FileInfo().Name(), "."):
// Get file from archive.
rc, err := f.Open()
if err != nil {
return nil, err
}
// Write data to file
fw, _ := os.Create(absPath)
if err != nil {
return nil, err
}
_, err = io.Copy(fw, rc)
// Close files.
rc.Close()
fw.Close()
if err != nil {
return nil, err
}
// Set modify time.
os.Chtimes(absPath, f.ModTime(), f.ModTime())
}
}
var imports []string
// Check if need to check imports.
if isCheckImport {
for _, d := range dirs {
importPkgs, err := CheckImports(d, match["importPath"])
if err != nil {
return nil, err
}
imports = append(imports, importPkgs...)
}
}
/*fpath := appPath + "repo/tarballs/" + node.ImportPath + "-" + node.Value + ".zip"
// Save tarball.
if autoBackup && !utils.IsExist(fpath) {
os.MkdirAll(path.Dir(fpath)+"/", os.ModePerm)
f, err := os.Create(fpath)
if err != nil {
return nil, err
}
defer f.Close()
_, err = f.Write(p)
}*/
return imports, err
}
// GetBitbucketDoc downloads tarball from bitbucket.org.
func GetBitbucketDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
// Check version control.
if m := bitbucketEtagRe.FindStringSubmatch(node.Value); m != nil {
match["vcs"] = m[1]
} else {
var repo struct {
Scm string
}
if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil {
return nil, err
}
match["vcs"] = repo.Scm
}
// bundle and snapshot will have commit 'B' and 'S',
// but does not need to download dependencies.
isCheckImport := len(node.Value) == 0
switch {
case isCheckImport || len(node.Value) == 1:
// Get up-to-date version.
tags := make(map[string]string)
for _, nodeType := range []string{"branches", "tags"} {
var nodes map[string]struct {
Node string
}
if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil {
return nil, err
}
for t, n := range nodes {
tags[t] = n.Node
}
}
// Check revision tag.
var err error
match["tag"], match["commit"], err = bestTag(tags, defaultTags[match["vcs"]])
if err != nil {
return nil, err
}
node.Type = "commit"
node.Value = match["commit"]
case !isCheckImport: // Bundle or snapshot.
// Check downlaod type.
switch node.Type {
case "tag", "commit", "branch":
match["commit"] = node.Value
default:
return nil, errors.New("Unknown node type: " + node.Type)
}
}
// We use .tar.gz here.
// zip : https://bitbucket.org/{owner}/{repo}/get/{commit}.zip
// tarball : https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz
// Downlaod archive.
p, err := HttpGetBytes(client, expand("https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz", match), nil)
if err != nil {
return nil, err
}
projectPath := expand("bitbucket.org/{owner}/{repo}", match)
installPath := installGOPATH + "/src/" + projectPath
node.ImportPath = projectPath
// Remove old files.
os.RemoveAll(installPath + "/")
// Create destination directory.
os.MkdirAll(installPath+"/", os.ModePerm)
gzr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return nil, err
}
defer gzr.Close()
tr := tar.NewReader(gzr)
isCodeOnly := cmdFlags["-c"]
var autoPath string // Auto path is the root path that generated by bitbucket.org.
// Get source file data.
dirs := make([]string, 0, 5)
for {
h, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
fn := h.FileInfo().Name()
// In case that we find directory, usually we should not.
if strings.HasSuffix(fn, "/") {
continue
}
// Check root path.
if len(autoPath) == 0 {
autoPath = fn[:strings.Index(fn, "/")]
}
absPath := strings.Replace(fn, autoPath, installPath, 1)
// Create diretory before create file.
dir := path.Dir(absPath)
if !checkDir(dir, dirs) && !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
dirs = append(dirs, dir)
os.MkdirAll(dir+"/", os.ModePerm)
}
if isCodeOnly && !utils.IsDocFile(path.Base(absPath)) {
continue
} else if strings.HasPrefix(fn, ".") {
continue
}
// Get data from archive.
fbytes := make([]byte, h.Size)
if _, err := io.ReadFull(tr, fbytes); err != nil {
return nil, err
}
// Write data to file
fw, err := os.Create(absPath)
if err != nil {
return nil, err
}
_, err = fw.Write(fbytes)
fw.Close()
if err != nil {
return nil, err
}
// Set modify time.
os.Chtimes(absPath, h.AccessTime, h.ModTime)
}
var imports []string
// Check if need to check imports.
if isCheckImport {
for _, d := range dirs {
importPkgs, err := CheckImports(d+"/", match["importPath"])
if err != nil {
return nil, err
}
imports = append(imports, importPkgs...)
}
}
return imports, err
}