// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
const (
put = iota
del
touch
)
action := put
etag := ""
if pdoc != nil {
etag = pdoc.Etag
}
var d time.Duration
if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
// Go source tree mirror.
pdoc = nil
action = del
} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
// Go Frontend source tree mirror.
pdoc = nil
action = del
} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
// Suffix of path matches another package.
pdoc = nil
action = del
} else {
t := time.Now()
pdocNew, err := doc.Get(httpClient, path, etag)
d = time.Since(t) / time.Millisecond
// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
httpTransport.CloseIdleConnections()
switch err {
case doc.ErrPackageNotModified:
action = touch
case doc.ErrPackageNotFound:
pdoc = nil
action = del
case nil:
pdoc = pdocNew
action = put
default:
log.Printf("%s error %q %q %dms %v", source, path, etag, d, err)
return nil, err
}
if pdoc != nil && !hasSubdirs {
if pdoc.Name == "" {
// Handle directories with no child directories as not found.
pdoc = nil
action = del
} else if pdoc.IsCmd && pdoc.Synopsis == "" {
///Don't store commands with no documentation and no children.
action = del
}
}
}
switch action {
case put:
log.Printf("%s put %q %q %dms", source, path, etag, d)
if err := db.Put(pdoc); err != nil {
log.Printf("ERROR db.Put(%q): %v", path, err)
}
case touch:
log.Printf("%s touch %q %q %dms", source, path, etag, d)
if err := db.TouchLastCrawl(path); err != nil {
log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
}
case del:
log.Printf("%s delete %q %q %dms", source, path, etag, d)
if err := db.Delete(path); err != nil {
log.Printf("ERROR db.Delete(%q): %v", path, err)
}
default:
panic("should not get here")
}
return pdoc, nil
}
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
etag := ""
if pdoc != nil {
etag = pdoc.Etag
}
var d time.Duration
var err error
if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
// Go source tree mirror.
pdoc = nil
err = doc.NotFoundError{"Go source tree mirror."}
} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
// Go Frontend source tree mirror.
pdoc = nil
err = doc.NotFoundError{"Go Frontend source tree mirror."}
} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
pdoc = nil
err = doc.NotFoundError{"Copy of other project."}
} else if blocked, e := db.IsBlocked(path); blocked && e == nil {
pdoc = nil
err = doc.NotFoundError{"Blocked."}
} else {
t := time.Now()
var pdocNew *doc.Package
pdocNew, err = doc.Get(httpClient, path, etag)
d = time.Since(t) / time.Millisecond
// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
httpTransport.CloseIdleConnections()
if err != doc.ErrNotModified {
pdoc = pdocNew
}
}
switch {
case err == nil:
log.Printf("%s put %q %q -> %q %dms", source, path, etag, pdoc.Etag, d)
if err := db.Put(pdoc); err != nil {
log.Printf("ERROR db.Put(%q): %v", path, err)
}
case err == doc.ErrNotModified:
log.Printf("%s touch %q %q %dms", source, path, etag, d)
if err := db.TouchLastCrawl(pdoc); err != nil {
log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
}
case doc.IsNotFound(err):
pdoc = nil
log.Printf("%s delete %q %s %dms", source, path, err.Error(), d)
if err := db.Delete(path); err != nil {
log.Printf("ERROR db.Delete(%q): %v", path, err)
}
default:
log.Printf("%s error %q %q %dms %v", source, path, etag, d, err)
return nil, err
}
return pdoc, nil
}