func runHandler(resp http.ResponseWriter, req *http.Request, fn func(resp http.ResponseWriter, req *http.Request) error, errfn httputil.Error) { defer func() { if rv := recover(); rv != nil { err := errors.New("handler panic") logError(req, err, rv) errfn(resp, req, http.StatusInternalServerError, err) } }() if s := req.Header.Get("X-Real-Ip"); s != "" && httputil.StripPort(req.RemoteAddr) == "127.0.0.1" { req.RemoteAddr = s } req.Body = http.MaxBytesReader(resp, req.Body, 2048) req.ParseForm() var rb httputil.ResponseBuffer err := fn(&rb, req) if err == nil { rb.WriteTo(resp) } else if e, ok := err.(*httpError); ok { if e.status >= 500 { logError(req, err, nil) } errfn(resp, req, e.status, e.err) } else if gosrc.IsNotFound(err) { errfn(resp, req, http.StatusNotFound, nil) } else { logError(req, err, nil) errfn(resp, req, http.StatusInternalServerError, err) } }
func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) err := f(w, r) if err == nil { return } else if gosrc.IsNotFound(err) { writeErrorResponse(w, 404) } else if e, ok := err.(*gosrc.RemoteError); ok { c.Infof("Remote error %s: %v", e.Host, e) writeResponse(w, 500, errorTemplate, fmt.Sprintf("Error accessing %s.", e.Host)) } else if err != nil { c.Errorf("Internal error %v", err) writeErrorResponse(w, 500) } }
func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) err := f(w, r) if err == nil { return } else if gosrc.IsNotFound(err) { writeTextHeader(w, 400) io.WriteString(w, "Not Found.") } else if e, ok := err.(*gosrc.RemoteError); ok { writeTextHeader(w, 500) fmt.Fprintf(w, "Error accessing %s.\n%v", e.Host, e) c.Infof("Remote error %s: %v", e.Host, e) } else if e, ok := err.(presFileNotFoundError); ok { writeTextHeader(w, 200) io.WriteString(w, e.Error()) } else if err != nil { writeTextHeader(w, 500) io.WriteString(w, "Internal server error.") c.Errorf("Internal error %v", err) } }
func Get(client *http.Client, importPath string, etag string) (*Package, error) { const versionPrefix = PackageVersion + "-" if strings.HasPrefix(etag, versionPrefix) { etag = etag[len(versionPrefix):] } else { etag = "" } dir, err := gosrc.Get(client, importPath, etag) if err != nil { return nil, err } pdoc, err := newPackage(dir) if err != nil { return pdoc, err } if pdoc.Synopsis == "" && pdoc.Doc == "" && !pdoc.IsCmd && pdoc.Name != "" && dir.ImportPath == dir.ProjectRoot && len(pdoc.Errors) == 0 { project, err := gosrc.GetProject(client, dir.ResolvedPath) switch { case err == nil: pdoc.Synopsis = doc.Synopsis(project.Description) case gosrc.IsNotFound(err): // ok default: return nil, err } } return pdoc, nil }
func runHandler(resp http.ResponseWriter, req *http.Request, fn func(resp http.ResponseWriter, req *http.Request) error, errfn httputil.Error) { defer func() { if rv := recover(); rv != nil { err := errors.New("handler panic") logError(req, err, rv) errfn(resp, req, http.StatusInternalServerError, err) } }() if *trustProxy { // If running on GAE, use X-Appengine-User-Ip to identify real ip of requests. if s := req.Header.Get("X-Appengine-User-Ip"); s != "" { req.RemoteAddr = s } else if s := req.Header.Get("X-Real-Ip"); s != "" { req.RemoteAddr = s } } req.Body = http.MaxBytesReader(resp, req.Body, 2048) req.ParseForm() var rb httputil.ResponseBuffer err := fn(&rb, req) if err == nil { rb.WriteTo(resp) } else if e, ok := err.(*httpError); ok { if e.status >= 500 { logError(req, err, nil) } errfn(resp, req, e.status, e.err) } else if gosrc.IsNotFound(err) { errfn(resp, req, http.StatusNotFound, nil) } else { logError(req, err, nil) errfn(resp, req, http.StatusInternalServerError, err) } }
// getDoc gets the package documentation from the database or from the version // control system as needed. func getDoc(path string, requestType int) (*doc.Package, []database.Package, error) { if path == "-" { // A hack in the database package uses the path "-" to represent the // next document to crawl. Block "-" here so that requests to /- always // return not found. return nil, nil, &httpError{status: http.StatusNotFound} } pdoc, pkgs, nextCrawl, err := db.Get(path) if err != nil { return nil, nil, err } needsCrawl := false switch requestType { case queryRequest, apiRequest: needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0 case humanRequest: needsCrawl = nextCrawl.Before(time.Now()) case robotRequest: needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0 } if !needsCrawl { return pdoc, pkgs, nil } c := make(chan crawlResult, 1) go func() { pdoc, err := crawlDoc("web ", path, pdoc, len(pkgs) > 0, nextCrawl) c <- crawlResult{pdoc, err} }() timeout := *getTimeout if pdoc == nil { timeout = *firstGetTimeout } select { case cr := <-c: err = cr.err if err == nil { pdoc = cr.pdoc } case <-time.After(timeout): err = errUpdateTimeout } switch { case err == nil: return pdoc, pkgs, nil case gosrc.IsNotFound(err): return nil, nil, err case pdoc != nil: log.Printf("Serving %q from database after error getting doc: %v", path, err) return pdoc, pkgs, nil case err == errUpdateTimeout: log.Printf("Serving %q as not found after timeout getting doc", path) return nil, nil, &httpError{status: http.StatusNotFound} default: return nil, nil, err } }
func servePackage(resp http.ResponseWriter, req *http.Request) error { p := path.Clean(req.URL.Path) if strings.HasPrefix(p, "/pkg/") { p = p[len("/pkg"):] } if p != req.URL.Path { http.Redirect(resp, req, p, http.StatusMovedPermanently) return nil } if isView(req, "status.svg") { statusImageHandlerSVG.ServeHTTP(resp, req) return nil } if isView(req, "status.png") { statusImageHandlerPNG.ServeHTTP(resp, req) return nil } requestType := humanRequest if isRobot(req) { requestType = robotRequest } importPath := strings.TrimPrefix(req.URL.Path, "/") pdoc, pkgs, err := getDoc(importPath, requestType) if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" { // To prevent dumb clients from following redirect loops, respond with // status 404 if the target document is not found. if _, _, err := getDoc(e.Redirect, requestType); gosrc.IsNotFound(err) { return &httpError{status: http.StatusNotFound} } u := "/" + e.Redirect if req.URL.RawQuery != "" { u += "?" + req.URL.RawQuery } setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}}) http.Redirect(resp, req, u, http.StatusFound) return nil } if err != nil { return err } flashMessages := getFlashMessages(resp, req) if pdoc == nil { if len(pkgs) == 0 { return &httpError{status: http.StatusNotFound} } pdocChild, _, _, err := db.Get(pkgs[0].Path) if err != nil { return err } pdoc = &doc.Package{ ProjectName: pdocChild.ProjectName, ProjectRoot: pdocChild.ProjectRoot, ProjectURL: pdocChild.ProjectURL, ImportPath: importPath, } } switch { case len(req.Form) == 0: importerCount := 0 if pdoc.Name != "" { importerCount, err = db.ImporterCount(importPath) if err != nil { return err } } etag := httpEtag(pdoc, pkgs, importerCount, flashMessages) status := http.StatusOK if req.Header.Get("If-None-Match") == etag { status = http.StatusNotModified } if requestType == humanRequest && pdoc.Name != "" && // not a directory pdoc.ProjectRoot != "" && // not a standard package !pdoc.IsCmd && len(pdoc.Errors) == 0 && !popularLinkReferral(req) { if err := db.IncrementPopularScore(pdoc.ImportPath); err != nil { log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err) } } if gceLogger != nil { gceLogger.LogEvent(resp, req, nil) } template := "dir" switch { case pdoc.IsCmd: template = "cmd" case pdoc.Name != "": template = "pkg" } template += templateExt(req) return executeTemplate(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{ "flashMessages": flashMessages, "pkgs": pkgs, "pdoc": newTDoc(pdoc), "importerCount": importerCount, }) case isView(req, "imports"): if pdoc.Name == "" { break } pkgs, err = db.Packages(pdoc.Imports) if err != nil { return err } return executeTemplate(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{ "flashMessages": flashMessages, "pkgs": pkgs, "pdoc": newTDoc(pdoc), }) case isView(req, "tools"): proto := "http" if req.Host == "godoc.org" { proto = "https" } return executeTemplate(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{ "flashMessages": flashMessages, "uri": fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath), "pdoc": newTDoc(pdoc), }) case isView(req, "importers"): if pdoc.Name == "" { break } pkgs, err = db.Importers(importPath) if err != nil { return err } template := "importers.html" if requestType == robotRequest { // Hide back links from robots. template = "importers_robot.html" } return executeTemplate(resp, template, http.StatusOK, nil, map[string]interface{}{ "flashMessages": flashMessages, "pkgs": pkgs, "pdoc": newTDoc(pdoc), }) case isView(req, "import-graph"): if requestType == robotRequest { return &httpError{status: http.StatusForbidden} } if pdoc.Name == "" { break } hide := database.ShowAllDeps switch req.Form.Get("hide") { case "1": hide = database.HideStandardDeps case "2": hide = database.HideStandardAll } pkgs, edges, err := db.ImportGraph(pdoc, hide) if err != nil { return err } b, err := renderGraph(pdoc, pkgs, edges) if err != nil { return err } return executeTemplate(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{ "flashMessages": flashMessages, "svg": template.HTML(b), "pdoc": newTDoc(pdoc), "hide": hide, }) case isView(req, "play"): u, err := playURL(pdoc, req.Form.Get("play"), req.Header.Get("X-AppEngine-Country")) if err != nil { return err } http.Redirect(resp, req, u, http.StatusMovedPermanently) return nil case req.Form.Get("view") != "": // Redirect deprecated view= queries. var q string switch view := req.Form.Get("view"); view { case "imports", "importers": q = view case "import-graph": if req.Form.Get("hide") == "1" { q = "import-graph&hide=1" } else { q = "import-graph" } } if q != "" { u := *req.URL u.RawQuery = q http.Redirect(resp, req, u.String(), http.StatusMovedPermanently) return nil } } return &httpError{status: http.StatusNotFound} }
// crawlDoc fetches the package documentation from the VCS and updates the database. func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) { message := []interface{}{source} defer func() { message = append(message, importPath) log.Println(message...) }() if !nextCrawl.IsZero() { d := time.Since(nextCrawl) / time.Hour if d > 0 { message = append(message, "late:", int64(d)) } } etag := "" if pdoc != nil { etag = pdoc.Etag message = append(message, "etag:", etag) } start := time.Now() var err error if strings.HasPrefix(importPath, "code.google.com/p/go.") { // Old import path for Go sub-repository. pdoc = nil err = gosrc.NotFoundError{Message: "old Go sub-repo", Redirect: "golang.org/x/" + importPath[len("code.google.com/p/go."):]} } else if blocked, e := db.IsBlocked(importPath); blocked && e == nil { pdoc = nil err = gosrc.NotFoundError{Message: "blocked."} } else if testdataPat.MatchString(importPath) { pdoc = nil err = gosrc.NotFoundError{Message: "testdata."} } else { var pdocNew *doc.Package pdocNew, err = doc.Get(httpClient, importPath, etag) message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond)) if err == nil && pdocNew.Name == "" && !hasSubdirs { pdoc = nil err = gosrc.NotFoundError{Message: "no Go files or subdirs"} } else if err != gosrc.ErrNotModified { pdoc = pdocNew } } nextCrawl = start.Add(*maxAge) switch { case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0): nextCrawl = start.Add(*maxAge * 7) case strings.HasPrefix(importPath, "gist.github.com/"): // Don't spend time on gists. It's silly thing to do. nextCrawl = start.Add(*maxAge * 30) } switch { case err == nil: message = append(message, "put:", pdoc.Etag) if err := db.Put(pdoc, nextCrawl, false); err != nil { log.Printf("ERROR db.Put(%q): %v", importPath, err) } return pdoc, nil case err == gosrc.ErrNotModified: message = append(message, "touch") if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil { log.Printf("ERROR db.SetNextCrawlEtag(%q): %v", importPath, err) } return pdoc, nil case gosrc.IsNotFound(err): message = append(message, "notfound:", err) if err := db.Delete(importPath); err != nil { log.Printf("ERROR db.Delete(%q): %v", importPath, err) } return nil, err default: message = append(message, "ERROR:", err) return nil, err } }
func servePackage(resp http.ResponseWriter, req *http.Request) error { p := path.Clean(req.URL.Path) if strings.HasPrefix(p, "/pkg/") { p = p[len("/pkg"):] } if p != req.URL.Path { http.Redirect(resp, req, p, http.StatusMovedPermanently) return nil } if isView(req, "status.svg") { statusImageHandlerSVG.ServeHTTP(resp, req) return nil } if isView(req, "status.png") { statusImageHandlerPNG.ServeHTTP(resp, req) return nil } requestType := humanRequest if isRobot(req) { requestType = robotRequest } importPath := strings.TrimPrefix(req.URL.Path, "/") pdoc, pkgs, err := getDoc(importPath, requestType) if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" { // To prevent dumb clients from following redirect loops, respond with // status 404 if the target document is not found. if _, _, err := getDoc(e.Redirect, requestType); gosrc.IsNotFound(err) { return &httpError{status: http.StatusNotFound} } u := "/" + e.Redirect if req.URL.RawQuery != "" { u += "?" + req.URL.RawQuery } http.Redirect(resp, req, u, http.StatusMovedPermanently) return nil } if err != nil { return err } if pdoc == nil { if len(pkgs) == 0 { return &httpError{status: http.StatusNotFound} } pdocChild, _, _, err := db.Get(pkgs[0].Path) if err != nil { return err } pdoc = &doc.Package{ ProjectName: pdocChild.ProjectName, ProjectRoot: pdocChild.ProjectRoot, ProjectURL: pdocChild.ProjectURL, ImportPath: importPath, } } switch { case len(req.Form) == 0: importerCount := 0 if pdoc.Name != "" { importerCount, err = db.ImporterCount(importPath) if err != nil { return err } } etag := httpEtag(pdoc, pkgs, importerCount) status := http.StatusOK if req.Header.Get("If-None-Match") == etag { status = http.StatusNotModified } if requestType == humanRequest && pdoc.Name != "" && // not a directory pdoc.ProjectRoot != "" && // not a standard package !pdoc.IsCmd && len(pdoc.Errors) == 0 && !popularLinkReferral(req) { if err := db.IncrementPopularScore(pdoc.ImportPath); err != nil { log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err) } } template := "dir" switch { case pdoc.IsCmd: template = "cmd" case pdoc.Name != "": template = "pkg" } template += templateExt(req) if srcFiles[importPath+"/_sourceMap"] != nil { for _, f := range pdoc.Files { if srcFiles[importPath+"/"+f.Name] != nil { f.URL = fmt.Sprintf("/%s?file=%s", importPath, f.Name) pdoc.LineFmt = "%s#L%d" } } } return executeTemplate(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{ "pkgs": pkgs, "pdoc": newTDoc(pdoc), "importerCount": importerCount, }) case isView(req, "imports"): if pdoc.Name == "" { break } pkgs, err = db.Packages(pdoc.Imports) if err != nil { return err } return executeTemplate(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{ "pkgs": pkgs, "pdoc": newTDoc(pdoc), }) case isView(req, "tools"): proto := "http" if req.Host == "godoc.org" { proto = "https" } return executeTemplate(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{ "uri": fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath), "pdoc": newTDoc(pdoc), }) case isView(req, "redir"): if srcFiles == nil { break } f := srcFiles[importPath+"/_sourceMap"] if f == nil { break } rc, err := f.Open() if err != nil { return err } defer rc.Close() var sourceMap map[string]string if err := gob.NewDecoder(rc).Decode(&sourceMap); err != nil { return err } id := req.Form.Get("redir") fname := sourceMap[id] if fname == "" { break } http.Redirect(resp, req, fmt.Sprintf("?file=%s#%s", fname, id), http.StatusMovedPermanently) return nil case isView(req, "file"): if srcFiles == nil { break } fname := req.Form.Get("file") f := srcFiles[importPath+"/"+fname] if f == nil { break } r, err := f.Open() if err != nil { return err } defer r.Close() src := make([]byte, f.UncompressedSize64) if n, err := io.ReadFull(r, src); err != nil { return err } else { src = src[:n] } var url string for _, f := range pdoc.Files { if f.Name == fname { url = f.URL } } return executeTemplate(resp, "file.html", http.StatusOK, nil, map[string]interface{}{ "fname": fname, "url": url, "src": template.HTML(src), "pdoc": newTDoc(pdoc), }) case isView(req, "importers"): if pdoc.Name == "" { break } pkgs, err = db.Importers(importPath) if err != nil { return err } template := "importers.html" if requestType == robotRequest { // Hide back links from robots. template = "importers_robot.html" } return executeTemplate(resp, template, http.StatusOK, nil, map[string]interface{}{ "pkgs": pkgs, "pdoc": newTDoc(pdoc), }) case isView(req, "import-graph"): if pdoc.Name == "" { break } hide := database.ShowAllDeps switch req.Form.Get("hide") { case "1": hide = database.HideStandardDeps case "2": hide = database.HideStandardAll } pkgs, edges, err := db.ImportGraph(pdoc, hide) if err != nil { return err } b, err := renderGraph(pdoc, pkgs, edges) if err != nil { return err } return executeTemplate(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{ "svg": template.HTML(b), "pdoc": newTDoc(pdoc), "hide": hide, }) case isView(req, "play"): u, err := playURL(pdoc, req.Form.Get("play")) if err != nil { return err } http.Redirect(resp, req, u, http.StatusMovedPermanently) return nil case req.Form.Get("view") != "": // Redirect deprecated view= queries. var q string switch view := req.Form.Get("view"); view { case "imports", "importers": q = view case "import-graph": if req.Form.Get("hide") == "1" { q = "import-graph&hide=1" } else { q = "import-graph" } } if q != "" { u := *req.URL u.RawQuery = q http.Redirect(resp, req, u.String(), http.StatusMovedPermanently) return nil } } return &httpError{status: http.StatusNotFound} }
// crawlDoc fetches the package documentation from the VCS and updates the database. func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) { message := []interface{}{source} defer func() { message = append(message, importPath) log.Println(message...) }() if !nextCrawl.IsZero() { d := time.Since(nextCrawl) / time.Hour if d > 0 { message = append(message, "late:", int64(d)) } } etag := "" if pdoc != nil { etag = pdoc.Etag message = append(message, "etag:", etag) } start := time.Now() var err error if i := strings.Index(importPath, "/src/pkg/"); i > 0 && gosrc.IsGoRepoPath(importPath[i+len("/src/pkg/"):]) { // Go source tree mirror. pdoc = nil err = gosrc.NotFoundError{Message: "Go source tree mirror."} } else if i := strings.Index(importPath, "/libgo/go/"); i > 0 && gosrc.IsGoRepoPath(importPath[i+len("/libgo/go/"):]) { // Go Frontend source tree mirror. pdoc = nil err = gosrc.NotFoundError{Message: "Go Frontend source tree mirror."} } else if m := nestedProjectPat.FindStringIndex(importPath); m != nil && exists(importPath[m[0]+1:]) { pdoc = nil err = gosrc.NotFoundError{Message: "Copy of other project."} } else if blocked, e := db.IsBlocked(importPath); blocked && e == nil { pdoc = nil err = gosrc.NotFoundError{Message: "Blocked."} } else { var pdocNew *doc.Package pdocNew, err = doc.Get(httpClient, importPath, etag) message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond)) if err == nil && pdocNew.Name == "" && !hasSubdirs { pdoc = nil err = gosrc.NotFoundError{Message: "No Go files or subdirs"} } else if err != gosrc.ErrNotModified { pdoc = pdocNew } } nextCrawl = start.Add(*maxAge) switch { case strings.HasPrefix(importPath, "github.com/") || (pdoc != nil && len(pdoc.Errors) > 0): nextCrawl = start.Add(*maxAge * 7) case strings.HasPrefix(importPath, "gist.github.com/"): // Don't spend time on gists. It's silly thing to do. nextCrawl = start.Add(*maxAge * 30) } switch { case err == nil: message = append(message, "put:", pdoc.Etag) if err := db.Put(pdoc, nextCrawl, false); err != nil { log.Printf("ERROR db.Put(%q): %v", importPath, err) } case err == gosrc.ErrNotModified: message = append(message, "touch") if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil { log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err) } case gosrc.IsNotFound(err): message = append(message, "notfound:", err) if err := db.Delete(importPath); err != nil { log.Printf("ERROR db.Delete(%q): %v", importPath, err) } default: message = append(message, "ERROR:", err) return nil, err } return pdoc, nil }