func newDocGet(httpClient doc.HttpClient, pkg string,
etag string) (p *doc.Package, err error) {
gp, err := glgddo.Get(httpClient.(*BlackRequest).client.(*http.Client),
pkg, etag)
if err != nil {
if err == gosrc.ErrNotModified {
err = doc.ErrNotModified
}
return nil, err
}
return &doc.Package{
ImportPath: gp.ImportPath,
ProjectRoot: gp.ProjectRoot,
ProjectName: gp.ProjectName,
ProjectURL: gp.ProjectURL,
Errors: gp.Errors,
References: gp.References,
VCS: gp.VCS,
Updated: gp.Updated,
Etag: gp.Etag,
Name: gp.Name,
Synopsis: gp.Synopsis,
Doc: gp.Doc,
IsCmd: gp.IsCmd,
Truncated: gp.Truncated,
GOOS: gp.GOOS,
GOARCH: gp.GOARCH,
LineFmt: gp.LineFmt,
BrowseURL: gp.BrowseURL,
SourceSize: gp.SourceSize,
TestSourceSize: gp.TestSourceSize,
Imports: gp.Imports,
TestImports: gp.TestImports,
XTestImports: gp.XTestImports,
StarCount: -1,
}, nil
// return nil, nil
}
func main() {
flag.Parse()
if len(flag.Args()) != 1 {
log.Fatal("Usage: go run print.go importPath")
}
path := flag.Args()[0]
var (
pdoc *doc.Package
err error
)
if *local {
gosrc.SetLocalDevMode(os.Getenv("GOPATH"))
}
pdoc, err = doc.Get(http.DefaultClient, path, *etag)
//}
if err != nil {
log.Fatal(err)
}
spew.Dump(pdoc)
}
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
message := []interface{}{source}
defer func() {
message = append(message, importPath)
log.Println(message...)
}()
if !nextCrawl.IsZero() {
d := time.Since(nextCrawl) / time.Hour
if d > 0 {
message = append(message, "late:", int64(d))
}
}
etag := ""
if pdoc != nil {
etag = pdoc.Etag
message = append(message, "etag:", etag)
}
start := time.Now()
var err error
if strings.HasPrefix(importPath, "code.google.com/p/go.") {
// Old import path for Go sub-repository.
pdoc = nil
err = gosrc.NotFoundError{Message: "old Go sub-repo", Redirect: "golang.org/x/" + importPath[len("code.google.com/p/go."):]}
} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
pdoc = nil
err = gosrc.NotFoundError{Message: "blocked."}
} else if testdataPat.MatchString(importPath) {
pdoc = nil
err = gosrc.NotFoundError{Message: "testdata."}
} else {
var pdocNew *doc.Package
pdocNew, err = doc.Get(httpClient, importPath, etag)
message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
if err == nil && pdocNew.Name == "" && !hasSubdirs {
pdoc = nil
err = gosrc.NotFoundError{Message: "no Go files or subdirs"}
} else if err != gosrc.ErrNotModified {
pdoc = pdocNew
}
}
nextCrawl = start.Add(*maxAge)
switch {
case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
nextCrawl = start.Add(*maxAge * 7)
case strings.HasPrefix(importPath, "gist.github.com/"):
// Don't spend time on gists. It's silly thing to do.
nextCrawl = start.Add(*maxAge * 30)
}
switch {
case err == nil:
message = append(message, "put:", pdoc.Etag)
if err := db.Put(pdoc, nextCrawl, false); err != nil {
log.Printf("ERROR db.Put(%q): %v", importPath, err)
}
return pdoc, nil
case err == gosrc.ErrNotModified:
message = append(message, "touch")
if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil {
log.Printf("ERROR db.SetNextCrawlEtag(%q): %v", importPath, err)
}
return pdoc, nil
case gosrc.IsNotFound(err):
message = append(message, "notfound:", err)
if err := db.Delete(importPath); err != nil {
log.Printf("ERROR db.Delete(%q): %v", importPath, err)
}
return nil, err
default:
message = append(message, "ERROR:", err)
return nil, err
}
}
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
message := []interface{}{source}
defer func() {
message = append(message, importPath)
log.Println(message...)
}()
if !nextCrawl.IsZero() {
d := time.Since(nextCrawl) / time.Hour
if d > 0 {
message = append(message, "late:", int64(d))
}
}
etag := ""
if pdoc != nil {
etag = pdoc.Etag
message = append(message, "etag:", etag)
}
start := time.Now()
var err error
if strings.HasPrefix(importPath, "code.google.com/p/go.") {
// Old import path for Go sub-repository.
pdoc = nil
err = gosrc.NotFoundError{Message: "old Go sub-repo", Redirect: "golang.org/x/" + importPath[len("code.google.com/p/go."):]}
} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
pdoc = nil
err = gosrc.NotFoundError{Message: "blocked."}
} else if testdataPat.MatchString(importPath) {
pdoc = nil
err = gosrc.NotFoundError{Message: "testdata."}
} else {
var pdocNew *doc.Package
pdocNew, err = doc.Get(httpClient, importPath, etag)
message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
if err == nil && pdocNew.Name == "" && !hasSubdirs {
for _, e := range pdocNew.Errors {
message = append(message, "err:", e)
}
pdoc = nil
err = gosrc.NotFoundError{Message: "no Go files or subdirs"}
} else if _, ok := err.(gosrc.NotModifiedError); !ok {
pdoc = pdocNew
}
}
nextCrawl = start.Add(*maxAge)
switch {
case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
nextCrawl = start.Add(*maxAge * 7)
case strings.HasPrefix(importPath, "gist.github.com/"):
// Don't spend time on gists. It's silly thing to do.
nextCrawl = start.Add(*maxAge * 30)
}
if err == nil {
message = append(message, "put:", pdoc.Etag)
if err := put(pdoc, nextCrawl); err != nil {
log.Println(err)
}
return pdoc, nil
} else if e, ok := err.(gosrc.NotModifiedError); ok {
if pdoc.Status == gosrc.Active && !isActivePkg(importPath, e.Status) {
if e.Status == gosrc.NoRecentCommits {
e.Status = gosrc.Inactive
}
message = append(message, "archive", e)
pdoc.Status = e.Status
if err := db.Put(pdoc, nextCrawl, false); err != nil {
log.Printf("ERROR db.Put(%q): %v", importPath, err)
}
} else {
// Touch the package without updating and move on to next one.
message = append(message, "touch")
if err := db.SetNextCrawl(importPath, nextCrawl); err != nil {
log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err)
}
}
return pdoc, nil
} else if e, ok := err.(gosrc.NotFoundError); ok {
message = append(message, "notfound:", e)
if err := db.Delete(importPath); err != nil {
log.Printf("ERROR db.Delete(%q): %v", importPath, err)
}
return nil, e
} else {
message = append(message, "ERROR:", err)
return nil, err
}
}
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
message := []interface{}{source}
defer func() {
message = append(message, importPath)
log.Println(message...)
}()
if !nextCrawl.IsZero() {
d := time.Since(nextCrawl) / time.Hour
if d > 0 {
message = append(message, "late:", int64(d))
}
}
etag := ""
if pdoc != nil {
etag = pdoc.Etag
message = append(message, "etag:", etag)
}
start := time.Now()
var err error
if i := strings.Index(importPath, "/src/pkg/"); i > 0 && gosrc.IsGoRepoPath(importPath[i+len("/src/pkg/"):]) {
// Go source tree mirror.
pdoc = nil
err = gosrc.NotFoundError{Message: "Go source tree mirror."}
} else if i := strings.Index(importPath, "/libgo/go/"); i > 0 && gosrc.IsGoRepoPath(importPath[i+len("/libgo/go/"):]) {
// Go Frontend source tree mirror.
pdoc = nil
err = gosrc.NotFoundError{Message: "Go Frontend source tree mirror."}
} else if m := nestedProjectPat.FindStringIndex(importPath); m != nil && exists(importPath[m[0]+1:]) {
pdoc = nil
err = gosrc.NotFoundError{Message: "Copy of other project."}
} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
pdoc = nil
err = gosrc.NotFoundError{Message: "Blocked."}
} else {
var pdocNew *doc.Package
pdocNew, err = doc.Get(httpClient, importPath, etag)
message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
if err == nil && pdocNew.Name == "" && !hasSubdirs {
pdoc = nil
err = gosrc.NotFoundError{Message: "No Go files or subdirs"}
} else if err != gosrc.ErrNotModified {
pdoc = pdocNew
}
}
nextCrawl = start.Add(*maxAge)
switch {
case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0):
nextCrawl = start.Add(*maxAge * 7)
case strings.HasPrefix(importPath, "gist.github.com/"):
// Don't spend time on gists. It's silly thing to do.
nextCrawl = start.Add(*maxAge * 30)
}
switch {
case err == nil:
message = append(message, "put:", pdoc.Etag)
if err := db.Put(pdoc, nextCrawl, false); err != nil {
log.Printf("ERROR db.Put(%q): %v", importPath, err)
}
case err == gosrc.ErrNotModified:
message = append(message, "touch")
if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil {
log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err)
}
case gosrc.IsNotFound(err):
message = append(message, "notfound:", err)
if err := db.Delete(importPath); err != nil {
log.Printf("ERROR db.Delete(%q): %v", importPath, err)
}
default:
message = append(message, "ERROR:", err)
return nil, err
}
return pdoc, nil
}