Exemple #1
0
func serveAPIUpdate(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if r.Method != "POST" {
		http.Error(w, "Method not supported.", http.StatusMethodNotAllowed)
		return
	}
	importPath := r.FormValue("importPath")
	pdoc, err := doc.Get(urlfetch.Client(c), importPath, "")
	if err == nil || err == doc.ErrPackageNotFound {
		err = updatePackage(c, importPath, pdoc)
	}

	if err != nil {
		c.Errorf("Error %s", err.Error())
		io.WriteString(w, "INTERNAL ERROR\n")
	} else if pdoc == nil {
		io.WriteString(w, "NOT FOUND\n")
	} else {
		io.WriteString(w, "OK\n")
	}
}
Exemple #2
0
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
	const (
		put = iota
		del
		touch
	)
	action := put

	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
	}

	var d time.Duration

	if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
		// Go source tree mirror.
		pdoc = nil
		action = del
	} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
		// Go Frontend source tree mirror.
		pdoc = nil
		action = del
	} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
		// Suffix of path matches another package.
		pdoc = nil
		action = del
	} else {
		t := time.Now()
		pdocNew, err := doc.Get(httpClient, path, etag)
		d = time.Since(t) / time.Millisecond

		// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
		httpTransport.CloseIdleConnections()

		switch err {
		case doc.ErrPackageNotModified:
			action = touch
		case doc.ErrPackageNotFound:
			pdoc = nil
			action = del
		case nil:
			pdoc = pdocNew
			action = put
		default:
			log.Printf("%s error  %q %q %dms %v", source, path, etag, d, err)
			return nil, err
		}

		if pdoc != nil && !hasSubdirs {
			if pdoc.Name == "" {
				// Handle directories with no child directories as not found.
				pdoc = nil
				action = del
			} else if pdoc.IsCmd && pdoc.Synopsis == "" {
				///Don't store commands with no documentation and no children.
				action = del
			}
		}
	}

	switch action {
	case put:
		log.Printf("%s put    %q %q %dms", source, path, etag, d)
		if err := db.Put(pdoc); err != nil {
			log.Printf("ERROR db.Put(%q): %v", path, err)
		}
	case touch:
		log.Printf("%s touch  %q %q %dms", source, path, etag, d)
		if err := db.TouchLastCrawl(path); err != nil {
			log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
		}
	case del:
		log.Printf("%s delete %q %q %dms", source, path, etag, d)
		if err := db.Delete(path); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", path, err)
		}
	default:
		panic("should not get here")
	}

	return pdoc, nil
}
Exemple #3
0
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
	}

	var d time.Duration
	var err error

	if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
		// Go source tree mirror.
		pdoc = nil
		err = doc.NotFoundError{"Go source tree mirror."}
	} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
		// Go Frontend source tree mirror.
		pdoc = nil
		err = doc.NotFoundError{"Go Frontend source tree mirror."}
	} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
		pdoc = nil
		err = doc.NotFoundError{"Copy of other project."}
	} else if blocked, e := db.IsBlocked(path); blocked && e == nil {
		pdoc = nil
		err = doc.NotFoundError{"Blocked."}
	} else {
		t := time.Now()
		var pdocNew *doc.Package
		pdocNew, err = doc.Get(httpClient, path, etag)
		d = time.Since(t) / time.Millisecond

		// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
		httpTransport.CloseIdleConnections()

		if err != doc.ErrNotModified {
			pdoc = pdocNew
		}
	}

	switch {
	case err == nil:
		log.Printf("%s put    %q %q -> %q %dms", source, path, etag, pdoc.Etag, d)
		if err := db.Put(pdoc); err != nil {
			log.Printf("ERROR db.Put(%q): %v", path, err)
		}
	case err == doc.ErrNotModified:
		log.Printf("%s touch  %q %q %dms", source, path, etag, d)
		if err := db.TouchLastCrawl(pdoc); err != nil {
			log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
		}
	case doc.IsNotFound(err):
		pdoc = nil
		log.Printf("%s delete %q %s %dms", source, path, err.Error(), d)
		if err := db.Delete(path); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", path, err)
		}
	default:
		log.Printf("%s error  %q %q %dms %v", source, path, etag, d, err)
		return nil, err
	}

	return pdoc, nil
}
Exemple #4
0
func main() {
	if len(os.Args) != 2 {
		log.Fatal("Usage: go run print.go importPath")
	}
	dpkg, err := doc.Get(http.DefaultClient, os.Args[1], "")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("ImportPath:  ", dpkg.ImportPath)
	fmt.Println("ProjectRoot: ", dpkg.ProjectRoot)
	fmt.Println("ProjectName: ", dpkg.ProjectName)
	fmt.Println("ProjectURL:  ", dpkg.ProjectURL)
	fmt.Println("Updated:     ", dpkg.Updated)
	fmt.Println("Etag:        ", dpkg.Etag)
	fmt.Println("Name:        ", dpkg.Name)
	fmt.Println("IsCmd:       ", dpkg.IsCmd)
	fmt.Println("Synopsis:    ", dpkg.Synopsis)
	fmt.Println("Doc:         ", indent(dpkg.Doc, 14))
	fmt.Println("Errors:")
	for _, s := range dpkg.Errors {
		fmt.Println("    ", s)
	}
	fmt.Println("Files:")
	for _, f := range dpkg.Files {
		fmt.Println("    ", f)
	}
	fmt.Println("Imports:")
	for _, i := range dpkg.Imports {
		fmt.Println("    ", i)
	}
	fmt.Println("TestImports:")
	for _, i := range dpkg.TestImports {
		fmt.Println("    ", i)
	}
	for _, c := range dpkg.Consts {
		fmt.Println("Const:")
		fmt.Println("    Decl:  ", indent(c.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(c.Doc, 12))
		fmt.Println("    URL:   ", c.URL)
	}
	for _, c := range dpkg.Vars {
		fmt.Println("Var:")
		fmt.Println("    Decl:  ", indent(c.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(c.Doc, 12))
		fmt.Println("    URL:   ", c.URL)
	}
	for _, f := range dpkg.Funcs {
		fmt.Println("Func:")
		fmt.Println("    Decl:  ", indent(f.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(f.Doc, 12))
		fmt.Println("    URL:   ", f.URL)
	}
	for _, t := range dpkg.Types {
		fmt.Println("Type:")
		fmt.Println("    Decl:  ", indent(t.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(t.Doc, 12))
		fmt.Println("    URL:   ", t.URL)
		for _, f := range t.Funcs {
			fmt.Println("    Func:")
			fmt.Println("        Decl:  ", indent(f.Decl.Text, 16))
			fmt.Println("        Doc:   ", indent(f.Doc, 16))
			fmt.Println("        URL:   ", f.URL)
		}
		for _, m := range t.Methods {
			fmt.Println("    Method:")
			fmt.Println("        Decl:  ", indent(m.Decl.Text, 16))
			fmt.Println("        Doc:   ", indent(m.Doc, 16))
			fmt.Println("        URL:   ", m.URL)
		}
	}
}
Exemple #5
0
func main() {
	flag.Parse()
	if len(flag.Args()) != 1 {
		log.Fatal("Usage: go run print.go importPath")
	}
	var (
		pdoc *doc.Package
		err  error
	)
	if *local {
		pdoc, err = doc.GetDir(flag.Args()[0])
	} else {
		pdoc, err = doc.Get(http.DefaultClient, flag.Args()[0], *etag)
	}
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("ImportPath:  ", pdoc.ImportPath)
	fmt.Println("ProjectRoot: ", pdoc.ProjectRoot)
	fmt.Println("ProjectName: ", pdoc.ProjectName)
	fmt.Println("ProjectURL:  ", pdoc.ProjectURL)
	fmt.Println("BrowseURL :  ", pdoc.BrowseURL)
	fmt.Println("Updated:     ", pdoc.Updated)
	fmt.Println("Etag:        ", pdoc.Etag)
	fmt.Println("Name:        ", pdoc.Name)
	fmt.Println("IsCmd:       ", pdoc.IsCmd)
	fmt.Println("Synopsis:    ", pdoc.Synopsis)
	fmt.Println("Doc:         ", indent(pdoc.Doc, 14))
	fmt.Println("Errors:")
	fmt.Println("Examples:", len(pdoc.Examples))
	for _, s := range pdoc.Errors {
		fmt.Println("    ", s)
	}
	fmt.Println("Files:")
	for _, f := range pdoc.Files {
		fmt.Println("    ", f)
	}
	fmt.Println("Imports:")
	for _, i := range pdoc.Imports {
		fmt.Println("    ", i)
	}
	fmt.Println("TestImports:")
	for _, i := range pdoc.TestImports {
		fmt.Println("    ", i)
	}
	for _, c := range pdoc.Consts {
		fmt.Println("Const:")
		fmt.Println("    Decl:  ", indent(c.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(c.Doc, 12))
		fmt.Println("    URL:   ", c.URL)
	}
	for _, c := range pdoc.Vars {
		fmt.Println("Var:")
		fmt.Println("    Decl:  ", indent(c.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(c.Doc, 12))
		fmt.Println("    URL:   ", c.URL)
	}
	for _, f := range pdoc.Funcs {
		fmt.Println("Func:")
		fmt.Println("    Decl:  ", indent(f.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(f.Doc, 12))
		fmt.Println("    URL:   ", f.URL)
		fmt.Println("    Examples:", len(f.Examples))
	}
	for _, t := range pdoc.Types {
		fmt.Println("Type:")
		fmt.Println("    Decl:  ", indent(t.Decl.Text, 12))
		fmt.Println("    Doc:   ", indent(t.Doc, 12))
		fmt.Println("    URL:   ", t.URL)
		fmt.Println("    Examples:", len(t.Examples))
		for _, f := range t.Funcs {
			fmt.Println("    Func:")
			fmt.Println("        Decl:  ", indent(f.Decl.Text, 16))
			fmt.Println("        Doc:   ", indent(f.Doc, 16))
			fmt.Println("        URL:   ", f.URL)
			fmt.Println("        Examples:", len(f.Examples))
		}
		for _, m := range t.Methods {
			fmt.Println("    Method:")
			fmt.Println("        Decl:  ", indent(m.Decl.Text, 16))
			fmt.Println("        Doc:   ", indent(m.Doc, 16))
			fmt.Println("        URL:   ", m.URL)
			fmt.Println("        Examples:", len(m.Examples))
		}
	}
}
Exemple #6
0
// getDoc gets the package documentation and child packages for the given import path.
func getDoc(c appengine.Context, importPath string) (*doc.Package, []*Package, error) {

	// 1. Look for doc in cache.

	cacheKey := docKeyPrefix + importPath
	var pdoc *doc.Package
	item, err := cacheGet(c, cacheKey, &pdoc)
	switch err {
	case nil:
		pkgs, err := childPackages(c, pdoc.ProjectRoot, importPath)
		if err != nil {
			return nil, nil, err
		}
		return pdoc, pkgs, err
	case memcache.ErrCacheMiss:
		// OK
	default:
		return nil, nil, err
	}

	// 2. Look for doc in store.

	pdocSaved, etag, err := loadDoc(c, importPath)
	if err != nil {
		return nil, nil, err
	}

	// 3. Get documentation from the version control service and update
	// datastore and cache as needed.

	pdoc, err = doc.Get(urlfetch.Client(c), importPath, etag)
	c.Infof("doc.Get(%q, %q) -> %v", importPath, etag, err)

	switch err {
	case nil:
		if err := updatePackage(c, importPath, pdoc); err != nil {
			return nil, nil, err
		}
		item.Object = pdoc
		item.Expiration = time.Hour
		if err := cacheSet(c, item); err != nil {
			return nil, nil, err
		}
	case doc.ErrPackageNotFound:
		if err := updatePackage(c, importPath, nil); err != nil {
			return nil, nil, err
		}
		return nil, nil, doc.ErrPackageNotFound
	case doc.ErrPackageNotModified:
		pdoc = pdocSaved
	default:
		if pdocSaved == nil {
			return nil, nil, err
		}
		c.Errorf("Serving %s from store after error from VCS.", importPath)
		pdoc = pdocSaved
	}

	// 4. Find the child packages.

	pkgs, err := childPackages(c, pdoc.ProjectRoot, importPath)
	if err != nil {
		return nil, nil, err
	}

	// 5. Convert to not found if package is empty.

	if len(pkgs) == 0 && pdoc.Name == "" && len(pdoc.Errors) == 0 {
		return nil, nil, doc.ErrPackageNotFound
	}

	// 6. Done

	return pdoc, pkgs, nil
}