コード例 #1
0
ファイル: crawler.go プロジェクト: digideskio/gcse
func newDocGet(httpClient doc.HttpClient, pkg string,
	etag string) (p *doc.Package, err error) {
	gp, err := glgddo.Get(httpClient.(*BlackRequest).client.(*http.Client),
		pkg, etag)
	if err != nil {
		if err == gosrc.ErrNotModified {
			err = doc.ErrNotModified
		}
		return nil, err
	}
	return &doc.Package{
		ImportPath:  gp.ImportPath,
		ProjectRoot: gp.ProjectRoot,

		ProjectName: gp.ProjectName,

		ProjectURL: gp.ProjectURL,

		Errors: gp.Errors,

		References: gp.References,

		VCS: gp.VCS,

		Updated: gp.Updated,

		Etag: gp.Etag,

		Name: gp.Name,

		Synopsis: gp.Synopsis,
		Doc:      gp.Doc,

		IsCmd: gp.IsCmd,

		Truncated: gp.Truncated,

		GOOS:   gp.GOOS,
		GOARCH: gp.GOARCH,

		LineFmt:   gp.LineFmt,
		BrowseURL: gp.BrowseURL,

		SourceSize:     gp.SourceSize,
		TestSourceSize: gp.TestSourceSize,

		Imports:      gp.Imports,
		TestImports:  gp.TestImports,
		XTestImports: gp.XTestImports,

		StarCount: -1,
	}, nil
	//	return nil, nil
}
コード例 #2
0
ファイル: print.go プロジェクト: maddyonline/gddo
func main() {
	flag.Parse()
	if len(flag.Args()) != 1 {
		log.Fatal("Usage: go run print.go importPath")
	}
	path := flag.Args()[0]

	var (
		pdoc *doc.Package
		err  error
	)
	if *local {
		gosrc.SetLocalDevMode(os.Getenv("GOPATH"))
	}
	pdoc, err = doc.Get(http.DefaultClient, path, *etag)
	//}
	if err != nil {
		log.Fatal(err)
	}
	spew.Dump(pdoc)
}
コード例 #3
0
ファイル: crawl.go プロジェクト: jjmiv/gddo
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
	message := []interface{}{source}
	defer func() {
		message = append(message, importPath)
		log.Println(message...)
	}()

	if !nextCrawl.IsZero() {
		d := time.Since(nextCrawl) / time.Hour
		if d > 0 {
			message = append(message, "late:", int64(d))
		}
	}

	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
		message = append(message, "etag:", etag)
	}

	start := time.Now()
	var err error
	if strings.HasPrefix(importPath, "code.google.com/p/go.") {
		// Old import path for Go sub-repository.
		pdoc = nil
		err = gosrc.NotFoundError{Message: "old Go sub-repo", Redirect: "golang.org/x/" + importPath[len("code.google.com/p/go."):]}
	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
		pdoc = nil
		err = gosrc.NotFoundError{Message: "blocked."}
	} else if testdataPat.MatchString(importPath) {
		pdoc = nil
		err = gosrc.NotFoundError{Message: "testdata."}
	} else {
		var pdocNew *doc.Package
		pdocNew, err = doc.Get(httpClient, importPath, etag)
		message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
		if err == nil && pdocNew.Name == "" && !hasSubdirs {
			pdoc = nil
			err = gosrc.NotFoundError{Message: "no Go files or subdirs"}
		} else if err != gosrc.ErrNotModified {
			pdoc = pdocNew
		}
	}

	nextCrawl = start.Add(*maxAge)
	switch {
	case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
		nextCrawl = start.Add(*maxAge * 7)
	case strings.HasPrefix(importPath, "gist.github.com/"):
		// Don't spend time on gists. It's silly thing to do.
		nextCrawl = start.Add(*maxAge * 30)
	}

	switch {
	case err == nil:
		message = append(message, "put:", pdoc.Etag)
		if err := db.Put(pdoc, nextCrawl, false); err != nil {
			log.Printf("ERROR db.Put(%q): %v", importPath, err)
		}
		return pdoc, nil
	case err == gosrc.ErrNotModified:
		message = append(message, "touch")
		if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil {
			log.Printf("ERROR db.SetNextCrawlEtag(%q): %v", importPath, err)
		}
		return pdoc, nil
	case gosrc.IsNotFound(err):
		message = append(message, "notfound:", err)
		if err := db.Delete(importPath); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", importPath, err)
		}
		return nil, err
	default:
		message = append(message, "ERROR:", err)
		return nil, err
	}
}
コード例 #4
0
ファイル: crawl.go プロジェクト: golang/gddo
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
	message := []interface{}{source}
	defer func() {
		message = append(message, importPath)
		log.Println(message...)
	}()

	if !nextCrawl.IsZero() {
		d := time.Since(nextCrawl) / time.Hour
		if d > 0 {
			message = append(message, "late:", int64(d))
		}
	}

	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
		message = append(message, "etag:", etag)
	}

	start := time.Now()
	var err error
	if strings.HasPrefix(importPath, "code.google.com/p/go.") {
		// Old import path for Go sub-repository.
		pdoc = nil
		err = gosrc.NotFoundError{Message: "old Go sub-repo", Redirect: "golang.org/x/" + importPath[len("code.google.com/p/go."):]}
	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
		pdoc = nil
		err = gosrc.NotFoundError{Message: "blocked."}
	} else if testdataPat.MatchString(importPath) {
		pdoc = nil
		err = gosrc.NotFoundError{Message: "testdata."}
	} else {
		var pdocNew *doc.Package
		pdocNew, err = doc.Get(httpClient, importPath, etag)
		message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
		if err == nil && pdocNew.Name == "" && !hasSubdirs {
			for _, e := range pdocNew.Errors {
				message = append(message, "err:", e)
			}
			pdoc = nil
			err = gosrc.NotFoundError{Message: "no Go files or subdirs"}
		} else if _, ok := err.(gosrc.NotModifiedError); !ok {
			pdoc = pdocNew
		}
	}

	nextCrawl = start.Add(*maxAge)
	switch {
	case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
		nextCrawl = start.Add(*maxAge * 7)
	case strings.HasPrefix(importPath, "gist.github.com/"):
		// Don't spend time on gists. It's silly thing to do.
		nextCrawl = start.Add(*maxAge * 30)
	}

	if err == nil {
		message = append(message, "put:", pdoc.Etag)
		if err := put(pdoc, nextCrawl); err != nil {
			log.Println(err)
		}
		return pdoc, nil
	} else if e, ok := err.(gosrc.NotModifiedError); ok {
		if pdoc.Status == gosrc.Active && !isActivePkg(importPath, e.Status) {
			if e.Status == gosrc.NoRecentCommits {
				e.Status = gosrc.Inactive
			}
			message = append(message, "archive", e)
			pdoc.Status = e.Status
			if err := db.Put(pdoc, nextCrawl, false); err != nil {
				log.Printf("ERROR db.Put(%q): %v", importPath, err)
			}
		} else {
			// Touch the package without updating and move on to next one.
			message = append(message, "touch")
			if err := db.SetNextCrawl(importPath, nextCrawl); err != nil {
				log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err)
			}
		}
		return pdoc, nil
	} else if e, ok := err.(gosrc.NotFoundError); ok {
		message = append(message, "notfound:", e)
		if err := db.Delete(importPath); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", importPath, err)
		}
		return nil, e
	} else {
		message = append(message, "ERROR:", err)
		return nil, err
	}
}
コード例 #5
0
ファイル: crawl.go プロジェクト: sendgrid-ops/gddo
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
	message := []interface{}{source}
	defer func() {
		message = append(message, importPath)
		log.Println(message...)
	}()

	if !nextCrawl.IsZero() {
		d := time.Since(nextCrawl) / time.Hour
		if d > 0 {
			message = append(message, "late:", int64(d))
		}
	}

	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
		message = append(message, "etag:", etag)
	}

	start := time.Now()
	var err error
	if i := strings.Index(importPath, "/src/pkg/"); i > 0 && gosrc.IsGoRepoPath(importPath[i+len("/src/pkg/"):]) {
		// Go source tree mirror.
		pdoc = nil
		err = gosrc.NotFoundError{Message: "Go source tree mirror."}
	} else if i := strings.Index(importPath, "/libgo/go/"); i > 0 && gosrc.IsGoRepoPath(importPath[i+len("/libgo/go/"):]) {
		// Go Frontend source tree mirror.
		pdoc = nil
		err = gosrc.NotFoundError{Message: "Go Frontend source tree mirror."}
	} else if m := nestedProjectPat.FindStringIndex(importPath); m != nil && exists(importPath[m[0]+1:]) {
		pdoc = nil
		err = gosrc.NotFoundError{Message: "Copy of other project."}
	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
		pdoc = nil
		err = gosrc.NotFoundError{Message: "Blocked."}
	} else {
		var pdocNew *doc.Package
		pdocNew, err = doc.Get(httpClient, importPath, etag)
		message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
		if err == nil && pdocNew.Name == "" && !hasSubdirs {
			pdoc = nil
			err = gosrc.NotFoundError{Message: "No Go files or subdirs"}
		} else if err != gosrc.ErrNotModified {
			pdoc = pdocNew
		}
	}

	nextCrawl = start.Add(*maxAge)
	switch {
	case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
		nextCrawl = start.Add(*maxAge * 7)
	case strings.HasPrefix(importPath, "gist.github.com/"):
		// Don't spend time on gists. It's silly thing to do.
		nextCrawl = start.Add(*maxAge * 30)
	}

	switch {
	case err == nil:
		message = append(message, "put:", pdoc.Etag)
		if err := db.Put(pdoc, nextCrawl, false); err != nil {
			log.Printf("ERROR db.Put(%q): %v", importPath, err)
		}
	case err == gosrc.ErrNotModified:
		message = append(message, "touch")
		if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil {
			log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err)
		}
	case gosrc.IsNotFound(err):
		message = append(message, "notfound:", err)
		if err := db.Delete(importPath); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", importPath, err)
		}
	default:
		message = append(message, "ERROR:", err)
		return nil, err
	}

	return pdoc, nil
}