Beispiel #1
0
// GetLaunchpadDoc downloads tarball from launchpad.net.
func GetLaunchpadDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {

	if match["project"] != "" && match["series"] != "" {
		rc, err := httpGet(client, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match), nil)
		switch {
		case err == nil:
			rc.Close()
			// The structure of the import path is launchpad.net/{root}/{dir}.
		case isNotFound(err):
			// The structure of the import path is is launchpad.net/{project}/{dir}.
			match["repo"] = match["project"]
			match["dir"] = expand("{series}{dir}", match)
		default:
			return nil, err
		}
	}

	// bundle and snapshot will have commit 'B' and 'S',
	// but does not need to download dependencies.
	isCheckImport := len(node.Value) == 0

	var downloadPath string
	// Check if download with specific revision.
	if isCheckImport || len(node.Value) == 1 {
		downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match)
		node.Type = "commit"
	} else {
		downloadPath = expand("https://bazaar.launchpad.net/+branch/{repo}/tarball/"+node.Value, match)
	}

	// Scrape the repo browser to find the project revision and individual Go files.
	p, err := HttpGetBytes(client, downloadPath, nil)
	if err != nil {
		return nil, err
	}

	projectPath := expand("launchpad.net/{repo}", match)
	installPath := installGOPATH + "/src/" + projectPath
	node.ImportPath = projectPath

	// Remove old files.
	os.RemoveAll(installPath + "/")
	// Create destination directory.
	os.MkdirAll(installPath+"/", os.ModePerm)

	gzr, err := gzip.NewReader(bytes.NewReader(p))
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)

	isCodeOnly := cmdFlags["-c"]
	var autoPath string // Auto path is the root path that generated by bitbucket.org.
	// Get source file data.
	dirs := make([]string, 0, 5)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		fn := h.FileInfo().Name()
		// Check root path.
		if len(autoPath) == 0 {
			autoPath = fn[:strings.Index(fn, match["repo"])+len(match["repo"])]
		}
		absPath := strings.Replace(fn, autoPath, installPath, 1)

		switch {
		case h.FileInfo().IsDir(): // Directory.
			// Check if current directory is example.
			if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
				dirs = append(dirs, absPath)
			}
		case isCodeOnly && !utils.IsDocFile(path.Base(absPath)):
			continue
		case !strings.HasPrefix(fn, "."):
			// Create diretory before create file.
			os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm)

			// Get data from archive.
			fbytes := make([]byte, h.Size)
			if _, err := io.ReadFull(tr, fbytes); err != nil {
				return nil, err
			}

			// Write data to file
			fw, err := os.Create(absPath)
			if err != nil {
				return nil, err
			}

			_, err = fw.Write(fbytes)
			fw.Close()
			if err != nil {
				return nil, err
			}
		}
	}

	var imports []string

	// Check if need to check imports.
	if isCheckImport {
		for _, d := range dirs {
			importPkgs, err := CheckImports(d+"/", match["importPath"])
			if err != nil {
				return nil, err
			}
			imports = append(imports, importPkgs...)
		}
	}

	return imports, err
}
Beispiel #2
0
// GetBitbucketDoc downloads tarball from bitbucket.org.
func GetBitbucketDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
	// Check version control.
	if m := bitbucketEtagRe.FindStringSubmatch(node.Value); m != nil {
		match["vcs"] = m[1]
	} else {
		var repo struct {
			Scm string
		}
		if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil {
			return nil, err
		}
		match["vcs"] = repo.Scm
	}

	// bundle and snapshot will have commit 'B' and 'S',
	// but does not need to download dependencies.
	isCheckImport := len(node.Value) == 0

	switch {
	case isCheckImport || len(node.Value) == 1:
		// Get up-to-date version.
		tags := make(map[string]string)
		for _, nodeType := range []string{"branches", "tags"} {
			var nodes map[string]struct {
				Node string
			}
			if err := httpGetJSON(client, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil {
				return nil, err
			}
			for t, n := range nodes {
				tags[t] = n.Node
			}
		}

		// Check revision tag.
		var err error
		match["tag"], match["commit"], err = bestTag(tags, defaultTags[match["vcs"]])
		if err != nil {
			return nil, err
		}

		node.Type = "commit"
		node.Value = match["commit"]
	case !isCheckImport: // Bundle or snapshot.
		// Check downlaod type.
		switch node.Type {
		case "tag", "commit", "branch":
			match["commit"] = node.Value
		default:
			return nil, errors.New("Unknown node type: " + node.Type)
		}
	}

	// We use .tar.gz here.
	// zip : https://bitbucket.org/{owner}/{repo}/get/{commit}.zip
	// tarball : https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz

	// Downlaod archive.
	p, err := HttpGetBytes(client, expand("https://bitbucket.org/{owner}/{repo}/get/{commit}.tar.gz", match), nil)
	if err != nil {
		return nil, err
	}

	projectPath := expand("bitbucket.org/{owner}/{repo}", match)
	installPath := installGOPATH + "/src/" + projectPath
	node.ImportPath = projectPath

	// Remove old files.
	os.RemoveAll(installPath + "/")
	// Create destination directory.
	os.MkdirAll(installPath+"/", os.ModePerm)

	gzr, err := gzip.NewReader(bytes.NewReader(p))
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)

	isCodeOnly := cmdFlags["-c"]
	var autoPath string // Auto path is the root path that generated by bitbucket.org.
	// Get source file data.
	dirs := make([]string, 0, 5)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		fn := h.FileInfo().Name()

		// In case that we find directory, usually we should not.
		if strings.HasSuffix(fn, "/") {
			continue
		}

		// Check root path.
		if len(autoPath) == 0 {
			autoPath = fn[:strings.Index(fn, "/")]
		}
		absPath := strings.Replace(fn, autoPath, installPath, 1)

		// Create diretory before create file.
		dir := path.Dir(absPath)
		if !checkDir(dir, dirs) && !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
			dirs = append(dirs, dir)
			os.MkdirAll(dir+"/", os.ModePerm)
		}

		if isCodeOnly && !utils.IsDocFile(path.Base(absPath)) {
			continue
		} else if strings.HasPrefix(fn, ".") {
			continue
		}

		// Get data from archive.
		fbytes := make([]byte, h.Size)
		if _, err := io.ReadFull(tr, fbytes); err != nil {
			return nil, err
		}

		// Write data to file
		fw, err := os.Create(absPath)
		if err != nil {
			return nil, err
		}

		_, err = fw.Write(fbytes)
		fw.Close()
		if err != nil {
			return nil, err
		}

		// Set modify time.
		os.Chtimes(absPath, h.AccessTime, h.ModTime)
	}

	var imports []string

	// Check if need to check imports.
	if isCheckImport {
		for _, d := range dirs {
			importPkgs, err := CheckImports(d+"/", match["importPath"])
			if err != nil {
				return nil, err
			}
			imports = append(imports, importPkgs...)
		}
	}

	return imports, err
}
Beispiel #3
0
// GetGithubDoc downloads tarball from github.com.
func GetGithubDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
	match["cred"] = githubCred

	// JSON struct for github.com.
	var refs []*struct {
		Ref    string
		Url    string
		Object struct {
			Sha  string
			Type string
			Url  string
		}
	}

	// bundle and snapshot will have commit 'B' and 'S',
	// but does not need to download dependencies.
	isCheckImport := len(node.Value) == 0

	switch {
	case isCheckImport || len(node.Value) == 1:
		// Get up-to-date version.
		err := httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), &refs)
		if err != nil {
			return nil, err
		}

		tags := make(map[string]string)
		for _, ref := range refs {
			switch {
			case strings.HasPrefix(ref.Ref, "refs/heads/"):
				tags[ref.Ref[len("refs/heads/"):]] = ref.Object.Sha
			case strings.HasPrefix(ref.Ref, "refs/tags/"):
				tags[ref.Ref[len("refs/tags/"):]] = ref.Object.Sha
			}
		}

		// Check revision tag.
		match["tag"], match["sha"], err = bestTag(tags, "master")
		if err != nil {
			return nil, err
		}

		node.Type = "commit"
		node.Value = match["sha"]
	case !isCheckImport: // Bundle or snapshot.
		// Check downlaod type.
		switch node.Type {
		case "tag", "commit", "branch":
			match["sha"] = node.Value
		default:
			return nil, errors.New("Unknown node type: " + node.Type)
		}
	}

	// We use .zip here.
	// zip : https://github.com/{owner}/{repo}/archive/{sha}.zip
	// tarball : https://github.com/{owner}/{repo}/tarball/{sha}

	// Downlaod archive.
	p, err := HttpGetBytes(client, expand("https://github.com/{owner}/{repo}/archive/{sha}.zip", match), nil)
	if err != nil {
		return nil, err
	}

	shaName := expand("{repo}-{sha}", match)
	if node.Type == "tag" {
		shaName = strings.Replace(shaName, "-v", "-", 1)
	}

	projectPath := expand("github.com/{owner}/{repo}", match)
	installPath := installGOPATH + "/src/" + projectPath
	node.ImportPath = projectPath

	// Remove old files.
	os.RemoveAll(installPath + "/")
	// Create destination directory.
	os.MkdirAll(installPath+"/", os.ModePerm)

	r, err := zip.NewReader(bytes.NewReader(p), int64(len(p)))
	if err != nil {
		return nil, err
	}

	isCodeOnly := cmdFlags["-c"]
	dirs := make([]string, 0, 5)
	// Need to add root path because we cannot get from tarball.
	dirs = append(dirs, installPath+"/")
	for _, f := range r.File {
		absPath := strings.Replace(f.FileInfo().Name(), shaName, installPath, 1)
		// Create diretory before create file.
		os.MkdirAll(path.Dir(absPath)+"/", os.ModePerm)

	compareDir:
		switch {
		case strings.HasSuffix(absPath, "/"): // Directory.
			// Check if current directory is example.
			if !(!cmdFlags["-e"] && strings.Contains(absPath, "example")) {
				for _, d := range dirs {
					if d == absPath {
						break compareDir
					}
				}
				dirs = append(dirs, absPath)
			}
		case isCodeOnly && !utils.IsDocFile(path.Base(absPath)):
			continue
		case !strings.HasPrefix(f.FileInfo().Name(), "."):
			// Get file from archive.
			rc, err := f.Open()
			if err != nil {
				return nil, err
			}

			// Write data to file
			fw, _ := os.Create(absPath)
			if err != nil {
				return nil, err
			}

			_, err = io.Copy(fw, rc)
			// Close files.
			rc.Close()
			fw.Close()
			if err != nil {
				return nil, err
			}

			// Set modify time.
			os.Chtimes(absPath, f.ModTime(), f.ModTime())
		}
	}

	var imports []string

	// Check if need to check imports.
	if isCheckImport {
		for _, d := range dirs {
			importPkgs, err := CheckImports(d, match["importPath"])
			if err != nil {
				return nil, err
			}
			imports = append(imports, importPkgs...)
		}
	}

	/*fpath := appPath + "repo/tarballs/" + node.ImportPath + "-" + node.Value + ".zip"
	// Save tarball.
	if autoBackup && !utils.IsExist(fpath) {
		os.MkdirAll(path.Dir(fpath)+"/", os.ModePerm)
		f, err := os.Create(fpath)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		_, err = f.Write(p)
	}*/

	return imports, err
}
Beispiel #4
0
// GetGoogleDoc downloads raw files from code.google.com.
func GetGoogleDoc(client *http.Client, match map[string]string, installGOPATH string, node *Node, cmdFlags map[string]bool) ([]string, error) {
	setupGoogleMatch(match)
	// Check version control.
	if m := googleEtagRe.FindStringSubmatch(node.Value); m != nil {
		match["vcs"] = m[1]
	} else if err := getGoogleVCS(client, match); err != nil {
		return nil, err
	}

	// bundle and snapshot will have commit 'B' and 'S',
	// but does not need to download dependencies.
	isCheckImport := len(node.Value) == 0
	if len(node.Value) == 1 {
		node.Value = ""
	}

	rootPath := expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match)

	// Scrape the repo browser to find the project revision and individual Go files.
	p, err := HttpGetBytes(client, rootPath+"?r="+node.Value, nil)
	if err != nil {
		return nil, err
	}

	// Check revision tag.
	if m := googleRevisionRe.FindSubmatch(p); m == nil {
		return nil,
			errors.New("doc.GetGoogleDoc(): Could not find revision for " + match["importPath"])
	} else {
		node.Type = "commit"
		node.Value = string(m[1])
	}

	projectPath := expand("code.google.com/p/{repo}{dot}{subrepo}{dir}", match)
	installPath := installGOPATH + "/src/" + projectPath
	node.ImportPath = projectPath

	// Remove old files.
	os.RemoveAll(installPath + "/")
	// Create destination directory.
	os.MkdirAll(installPath+"/", os.ModePerm)

	isCodeOnly := cmdFlags["-c"]
	// Get source files in root path.
	files := make([]*source, 0, 5)
	for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
		fname := strings.Split(string(m[1]), "?")[0]
		if isCodeOnly && !utils.IsDocFile(fname) {
			continue
		} else if strings.HasPrefix(fname, ".") {
			continue
		}

		files = append(files, &source{
			name:   fname,
			rawURL: expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname) + "?r=" + node.Value,
		})
	}

	// Fetch files from VCS.
	if err := fetchFiles(client, files, nil); err != nil {
		return nil, err
	}

	// Save files.
	for _, f := range files {
		absPath := installPath + "/"

		// Create diretory before create file.
		os.MkdirAll(path.Dir(absPath), os.ModePerm)

		// Write data to file
		fw, err := os.Create(absPath + f.name)
		if err != nil {
			return nil, err
		}

		_, err = fw.Write(f.data)
		fw.Close()
		if err != nil {
			return nil, err
		}
	}

	dirs := make([]string, 0, 3)
	// Get subdirectories.
	for _, m := range googleDirRe.FindAllSubmatch(p, -1) {
		dirName := strings.Split(string(m[1]), "?")[0]
		if strings.HasSuffix(dirName, "/") {
			dirs = append(dirs, dirName)
		}
	}

	err = downloadFiles(client, match, rootPath, installPath+"/", node.Value, dirs)
	if err != nil {
		return nil, err
	}

	var imports []string

	// Check if need to check imports.
	if isCheckImport {
		dirs, err := utils.GetDirsInfo(installPath + "/")
		if err != nil {
			return nil, err
		}

		for _, d := range dirs {
			if d.IsDir() && !(!cmdFlags["-e"] && strings.Contains(d.Name(), "example")) {
				absPath := installPath + "/" + d.Name() + "/"
				importPkgs, err := CheckImports(absPath, match["importPath"])
				if err != nil {
					return nil, err
				}
				imports = append(imports, importPkgs...)
			}
		}
	}

	return imports, err
}