func serveAPIUpdate(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
if r.Method != "POST" {
http.Error(w, "Method not supported.", http.StatusMethodNotAllowed)
return
}
importPath := r.FormValue("importPath")
pdoc, err := doc.Get(urlfetch.Client(c), importPath, "")
if err == nil || err == doc.ErrPackageNotFound {
err = updatePackage(c, importPath, pdoc)
}
if err != nil {
c.Errorf("Error %s", err.Error())
io.WriteString(w, "INTERNAL ERROR\n")
} else if pdoc == nil {
io.WriteString(w, "NOT FOUND\n")
} else {
io.WriteString(w, "OK\n")
}
}
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
const (
put = iota
del
touch
)
action := put
etag := ""
if pdoc != nil {
etag = pdoc.Etag
}
var d time.Duration
if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
// Go source tree mirror.
pdoc = nil
action = del
} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
// Go Frontend source tree mirror.
pdoc = nil
action = del
} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
// Suffix of path matches another package.
pdoc = nil
action = del
} else {
t := time.Now()
pdocNew, err := doc.Get(httpClient, path, etag)
d = time.Since(t) / time.Millisecond
// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
httpTransport.CloseIdleConnections()
switch err {
case doc.ErrPackageNotModified:
action = touch
case doc.ErrPackageNotFound:
pdoc = nil
action = del
case nil:
pdoc = pdocNew
action = put
default:
log.Printf("%s error %q %q %dms %v", source, path, etag, d, err)
return nil, err
}
if pdoc != nil && !hasSubdirs {
if pdoc.Name == "" {
// Handle directories with no child directories as not found.
pdoc = nil
action = del
} else if pdoc.IsCmd && pdoc.Synopsis == "" {
///Don't store commands with no documentation and no children.
action = del
}
}
}
switch action {
case put:
log.Printf("%s put %q %q %dms", source, path, etag, d)
if err := db.Put(pdoc); err != nil {
log.Printf("ERROR db.Put(%q): %v", path, err)
}
case touch:
log.Printf("%s touch %q %q %dms", source, path, etag, d)
if err := db.TouchLastCrawl(path); err != nil {
log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
}
case del:
log.Printf("%s delete %q %q %dms", source, path, etag, d)
if err := db.Delete(path); err != nil {
log.Printf("ERROR db.Delete(%q): %v", path, err)
}
default:
panic("should not get here")
}
return pdoc, nil
}
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
etag := ""
if pdoc != nil {
etag = pdoc.Etag
}
var d time.Duration
var err error
if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
// Go source tree mirror.
pdoc = nil
err = doc.NotFoundError{"Go source tree mirror."}
} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
// Go Frontend source tree mirror.
pdoc = nil
err = doc.NotFoundError{"Go Frontend source tree mirror."}
} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
pdoc = nil
err = doc.NotFoundError{"Copy of other project."}
} else if blocked, e := db.IsBlocked(path); blocked && e == nil {
pdoc = nil
err = doc.NotFoundError{"Blocked."}
} else {
t := time.Now()
var pdocNew *doc.Package
pdocNew, err = doc.Get(httpClient, path, etag)
d = time.Since(t) / time.Millisecond
// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
httpTransport.CloseIdleConnections()
if err != doc.ErrNotModified {
pdoc = pdocNew
}
}
switch {
case err == nil:
log.Printf("%s put %q %q -> %q %dms", source, path, etag, pdoc.Etag, d)
if err := db.Put(pdoc); err != nil {
log.Printf("ERROR db.Put(%q): %v", path, err)
}
case err == doc.ErrNotModified:
log.Printf("%s touch %q %q %dms", source, path, etag, d)
if err := db.TouchLastCrawl(pdoc); err != nil {
log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
}
case doc.IsNotFound(err):
pdoc = nil
log.Printf("%s delete %q %s %dms", source, path, err.Error(), d)
if err := db.Delete(path); err != nil {
log.Printf("ERROR db.Delete(%q): %v", path, err)
}
default:
log.Printf("%s error %q %q %dms %v", source, path, etag, d, err)
return nil, err
}
return pdoc, nil
}
func main() {
if len(os.Args) != 2 {
log.Fatal("Usage: go run print.go importPath")
}
dpkg, err := doc.Get(http.DefaultClient, os.Args[1], "")
if err != nil {
log.Fatal(err)
}
fmt.Println("ImportPath: ", dpkg.ImportPath)
fmt.Println("ProjectRoot: ", dpkg.ProjectRoot)
fmt.Println("ProjectName: ", dpkg.ProjectName)
fmt.Println("ProjectURL: ", dpkg.ProjectURL)
fmt.Println("Updated: ", dpkg.Updated)
fmt.Println("Etag: ", dpkg.Etag)
fmt.Println("Name: ", dpkg.Name)
fmt.Println("IsCmd: ", dpkg.IsCmd)
fmt.Println("Synopsis: ", dpkg.Synopsis)
fmt.Println("Doc: ", indent(dpkg.Doc, 14))
fmt.Println("Errors:")
for _, s := range dpkg.Errors {
fmt.Println(" ", s)
}
fmt.Println("Files:")
for _, f := range dpkg.Files {
fmt.Println(" ", f)
}
fmt.Println("Imports:")
for _, i := range dpkg.Imports {
fmt.Println(" ", i)
}
fmt.Println("TestImports:")
for _, i := range dpkg.TestImports {
fmt.Println(" ", i)
}
for _, c := range dpkg.Consts {
fmt.Println("Const:")
fmt.Println(" Decl: ", indent(c.Decl.Text, 12))
fmt.Println(" Doc: ", indent(c.Doc, 12))
fmt.Println(" URL: ", c.URL)
}
for _, c := range dpkg.Vars {
fmt.Println("Var:")
fmt.Println(" Decl: ", indent(c.Decl.Text, 12))
fmt.Println(" Doc: ", indent(c.Doc, 12))
fmt.Println(" URL: ", c.URL)
}
for _, f := range dpkg.Funcs {
fmt.Println("Func:")
fmt.Println(" Decl: ", indent(f.Decl.Text, 12))
fmt.Println(" Doc: ", indent(f.Doc, 12))
fmt.Println(" URL: ", f.URL)
}
for _, t := range dpkg.Types {
fmt.Println("Type:")
fmt.Println(" Decl: ", indent(t.Decl.Text, 12))
fmt.Println(" Doc: ", indent(t.Doc, 12))
fmt.Println(" URL: ", t.URL)
for _, f := range t.Funcs {
fmt.Println(" Func:")
fmt.Println(" Decl: ", indent(f.Decl.Text, 16))
fmt.Println(" Doc: ", indent(f.Doc, 16))
fmt.Println(" URL: ", f.URL)
}
for _, m := range t.Methods {
fmt.Println(" Method:")
fmt.Println(" Decl: ", indent(m.Decl.Text, 16))
fmt.Println(" Doc: ", indent(m.Doc, 16))
fmt.Println(" URL: ", m.URL)
}
}
}
func main() {
flag.Parse()
if len(flag.Args()) != 1 {
log.Fatal("Usage: go run print.go importPath")
}
var (
pdoc *doc.Package
err error
)
if *local {
pdoc, err = doc.GetDir(flag.Args()[0])
} else {
pdoc, err = doc.Get(http.DefaultClient, flag.Args()[0], *etag)
}
if err != nil {
log.Fatal(err)
}
fmt.Println("ImportPath: ", pdoc.ImportPath)
fmt.Println("ProjectRoot: ", pdoc.ProjectRoot)
fmt.Println("ProjectName: ", pdoc.ProjectName)
fmt.Println("ProjectURL: ", pdoc.ProjectURL)
fmt.Println("BrowseURL : ", pdoc.BrowseURL)
fmt.Println("Updated: ", pdoc.Updated)
fmt.Println("Etag: ", pdoc.Etag)
fmt.Println("Name: ", pdoc.Name)
fmt.Println("IsCmd: ", pdoc.IsCmd)
fmt.Println("Synopsis: ", pdoc.Synopsis)
fmt.Println("Doc: ", indent(pdoc.Doc, 14))
fmt.Println("Errors:")
fmt.Println("Examples:", len(pdoc.Examples))
for _, s := range pdoc.Errors {
fmt.Println(" ", s)
}
fmt.Println("Files:")
for _, f := range pdoc.Files {
fmt.Println(" ", f)
}
fmt.Println("Imports:")
for _, i := range pdoc.Imports {
fmt.Println(" ", i)
}
fmt.Println("TestImports:")
for _, i := range pdoc.TestImports {
fmt.Println(" ", i)
}
for _, c := range pdoc.Consts {
fmt.Println("Const:")
fmt.Println(" Decl: ", indent(c.Decl.Text, 12))
fmt.Println(" Doc: ", indent(c.Doc, 12))
fmt.Println(" URL: ", c.URL)
}
for _, c := range pdoc.Vars {
fmt.Println("Var:")
fmt.Println(" Decl: ", indent(c.Decl.Text, 12))
fmt.Println(" Doc: ", indent(c.Doc, 12))
fmt.Println(" URL: ", c.URL)
}
for _, f := range pdoc.Funcs {
fmt.Println("Func:")
fmt.Println(" Decl: ", indent(f.Decl.Text, 12))
fmt.Println(" Doc: ", indent(f.Doc, 12))
fmt.Println(" URL: ", f.URL)
fmt.Println(" Examples:", len(f.Examples))
}
for _, t := range pdoc.Types {
fmt.Println("Type:")
fmt.Println(" Decl: ", indent(t.Decl.Text, 12))
fmt.Println(" Doc: ", indent(t.Doc, 12))
fmt.Println(" URL: ", t.URL)
fmt.Println(" Examples:", len(t.Examples))
for _, f := range t.Funcs {
fmt.Println(" Func:")
fmt.Println(" Decl: ", indent(f.Decl.Text, 16))
fmt.Println(" Doc: ", indent(f.Doc, 16))
fmt.Println(" URL: ", f.URL)
fmt.Println(" Examples:", len(f.Examples))
}
for _, m := range t.Methods {
fmt.Println(" Method:")
fmt.Println(" Decl: ", indent(m.Decl.Text, 16))
fmt.Println(" Doc: ", indent(m.Doc, 16))
fmt.Println(" URL: ", m.URL)
fmt.Println(" Examples:", len(m.Examples))
}
}
}
// getDoc gets the package documentation and child packages for the given import path.
func getDoc(c appengine.Context, importPath string) (*doc.Package, []*Package, error) {
// 1. Look for doc in cache.
cacheKey := docKeyPrefix + importPath
var pdoc *doc.Package
item, err := cacheGet(c, cacheKey, &pdoc)
switch err {
case nil:
pkgs, err := childPackages(c, pdoc.ProjectRoot, importPath)
if err != nil {
return nil, nil, err
}
return pdoc, pkgs, err
case memcache.ErrCacheMiss:
// OK
default:
return nil, nil, err
}
// 2. Look for doc in store.
pdocSaved, etag, err := loadDoc(c, importPath)
if err != nil {
return nil, nil, err
}
// 3. Get documentation from the version control service and update
// datastore and cache as needed.
pdoc, err = doc.Get(urlfetch.Client(c), importPath, etag)
c.Infof("doc.Get(%q, %q) -> %v", importPath, etag, err)
switch err {
case nil:
if err := updatePackage(c, importPath, pdoc); err != nil {
return nil, nil, err
}
item.Object = pdoc
item.Expiration = time.Hour
if err := cacheSet(c, item); err != nil {
return nil, nil, err
}
case doc.ErrPackageNotFound:
if err := updatePackage(c, importPath, nil); err != nil {
return nil, nil, err
}
return nil, nil, doc.ErrPackageNotFound
case doc.ErrPackageNotModified:
pdoc = pdocSaved
default:
if pdocSaved == nil {
return nil, nil, err
}
c.Errorf("Serving %s from store after error from VCS.", importPath)
pdoc = pdocSaved
}
// 4. Find the child packages.
pkgs, err := childPackages(c, pdoc.ProjectRoot, importPath)
if err != nil {
return nil, nil, err
}
// 5. Convert to not found if package is empty.
if len(pkgs) == 0 && pdoc.Name == "" && len(pdoc.Errors) == 0 {
return nil, nil, doc.ErrPackageNotFound
}
// 6. Done
return pdoc, pkgs, nil
}
