func crawl(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
conn := db.Pool.Get()
defer conn.Close()
paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
if err != nil {
log.Fatal(err)
}
fmt.Println("NEW")
for _, path := range paths {
fmt.Println(path)
}
paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
if err != nil {
log.Fatal(err)
}
fmt.Println("BAD")
for _, path := range paths {
fmt.Println(path)
}
}
func stats(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
_, err := database.New()
if err != nil {
log.Fatal(err)
}
log.Println("DONE")
}
func block(c *command) {
if len(c.flag.Args()) != 1 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
if err := db.Block(c.flag.Args()[0]); err != nil {
log.Fatal(err)
}
}
func popular(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
pkgs, err := db.PopularWithScores()
if err != nil {
log.Fatal(err)
}
for _, pkg := range pkgs {
fmt.Println(pkg.Path, pkg.Synopsis)
}
}
func reindex(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
var n int
err = db.Do(func(pi *database.PackageInfo) error {
n += 1
fix(pi.PDoc)
return db.Put(pi.PDoc, time.Time{})
})
if err != nil {
log.Fatal(err)
}
log.Printf("Updated %d documents", n)
}
func dangle(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
m := make(map[string]int)
err = db.Do(func(pi *database.PackageInfo) error {
m[pi.PDoc.ImportPath] |= 1
for _, p := range pi.PDoc.Imports {
if doc.IsValidPath(p) {
m[p] |= 2
}
}
for _, p := range pi.PDoc.TestImports {
if doc.IsValidPath(p) {
m[p] |= 2
}
}
for _, p := range pi.PDoc.XTestImports {
if doc.IsValidPath(p) {
m[p] |= 2
}
}
return nil
})
for p, v := range m {
if v == 2 {
fmt.Println(p)
}
}
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
if err := readSecrets(); err != nil {
log.Fatal(err)
}
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"interface.html", "common.html", "layout.html"},
{"index.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
{"opensearch.xml"},
}); err != nil {
log.Fatal(err)
}
if err := parsePresentTemplates([][]string{
{".article", "article.tmpl", "action.tmpl"},
{".slide", "slides.tmpl", "action.tmpl"},
}); err != nil {
log.Fatal(err)
}
present.PlayEnabled = true
var err error
db, err = database.New()
if err != nil {
log.Fatal(err)
}
if *crawlInterval > 0 {
go crawl(*crawlInterval)
}
if *githubInterval > 0 {
go crawlGithubUpdates(*githubInterval)
}
playScript, err := readPlayScript(*presentDir)
if err != nil {
log.Fatal(err)
}
staticConfig := &web.StaticConfig{
Header: web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
Directory: *assetsDir,
GzDirectory: *gzAssetsDir,
}
presentStaticConfig := &web.StaticConfig{
Header: web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
Directory: *presentDir,
}
h := web.NewHostRouter()
r := web.NewRouter()
r.Add("/").GetFunc(servePresentHome)
r.Add("/compile").PostFunc(serveCompile)
r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
r.Add("/play.js").Get(web.DataHandler(playScript, web.Header{web.HeaderContentType: {"text/javascript"}}))
r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
r.Add("/static/<path:.*>").Get(presentStaticConfig.DirectoryHandler("static"))
r.Add("/<path:.+>").GetFunc(servePresentation)
h.Add("talks.<:.*>", web.ErrorHandler(handlePresentError, web.FormAndCookieHandler(6000, false, r)))
r = web.NewRouter()
r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
r.Add("/search").GetFunc(serveAPISearch)
r.Add("/packages").GetFunc(serveAPIPackages)
h.Add("api.<:.*>", web.ErrorHandler(handleAPIError, web.FormAndCookieHandler(6000, false, r)))
r = web.NewRouter()
r.Add("/").GetFunc(serveHome)
r.Add("/-/about").GetFunc(serveAbout)
r.Add("/-/bot").GetFunc(serveBot)
r.Add("/-/opensearch.xml").GetFunc(serveOpenSearchDescription)
r.Add("/-/typeahead").GetFunc(serveTypeahead)
r.Add("/-/go").GetFunc(serveGoIndex)
r.Add("/-/index").GetFunc(serveIndex)
r.Add("/-/refresh").PostFunc(serveRefresh)
r.Add("/-/static/<path:.*>").Get(staticConfig.DirectoryHandler("static"))
r.Add("/a/index").Get(web.RedirectHandler("/-/index", 301))
r.Add("/about").Get(web.RedirectHandler("/-/about", 301))
r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
r.Add("/robots.txt").Get(staticConfig.FileHandler("robots.txt"))
r.Add("/BingSiteAuth.xml").Get(staticConfig.FileHandler("BingSiteAuth.xml"))
r.Add("/C").Get(web.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301))
r.Add("/<path:.+>").GetFunc(servePackage)
h.Add("<:.*>", web.ErrorHandler(handleError, web.FormAndCookieHandler(1000, false, r)))
listener, err := net.Listen("tcp", *httpAddr)
if err != nil {
log.Fatal("Listen", err)
return
}
defer listener.Close()
s := &server.Server{Listener: listener, Handler: h} // add logger
err = s.Serve()
if err != nil {
log.Fatal("Server", err)
}
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
if err := readSecrets(); err != nil {
log.Fatal(err)
}
if *srcZip != "" {
r, err := zip.OpenReader(*srcZip)
if err != nil {
log.Fatal(err)
}
for _, f := range r.File {
if strings.HasPrefix(f.Name, "root/") {
srcFiles[f.Name[len("root/"):]] = f
}
}
}
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"dir.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"file.html", "common.html", "layout.html"},
{"index.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"subrepo.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"dir.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
{"opensearch.xml"},
}); err != nil {
log.Fatal(err)
}
if err := parsePresentTemplates([][]string{
{".article", "article.tmpl", "action.tmpl"},
{".slide", "slides.tmpl", "action.tmpl"},
}); err != nil {
log.Fatal(err)
}
present.PlayEnabled = true
var err error
db, err = database.New()
if err != nil {
log.Fatal(err)
}
go runBackgroundTasks()
staticConfig := &web.StaticConfig{
Header: web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
Directory: *assetsDir,
GzDirectory: *gzAssetsDir,
}
presentStaticConfig := &web.StaticConfig{
Header: web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
Directory: *presentDir,
}
h := web.NewHostRouter()
r := web.NewRouter()
r.Add("/").GetFunc(servePresentHome)
r.Add("/compile").PostFunc(serveCompile)
r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
r.Add("/play.js").Get(dataHandler("play.js", "text/javascript", *presentDir, "js/jquery.js", "js/playground.js", "js/play.js"))
r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
r.Add("/static/<path:.*>").Get(presentStaticConfig.DirectoryHandler("static"))
if *redirGoTalks {
r.Add("/code.google.com/p/go.talks/<path:.+>").GetFunc(func(resp web.Response, req *web.Request) error {
return web.Redirect(resp, req, "http://talks.golang.org/"+req.RouteVars["path"], 301, nil)
})
}
r.Add("/<path:.+>").GetFunc(servePresentation)
h.Add("talks.<:.*>", web.ErrorHandler(handlePresentError, web.FormAndCookieHandler(6000, false, r)))
r = web.NewRouter()
r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
r.Add("/search").GetFunc(serveAPISearch)
r.Add("/packages").GetFunc(serveAPIPackages)
r.Add("/importers/<path:.+>").GetFunc(serveAPIImporters)
h.Add("api.<:.*>", web.ErrorHandler(handleAPIError, web.FormAndCookieHandler(6000, false, r)))
r = web.NewRouter()
r.Add("/-/site.js").Get(dataHandler("site.js", "text/javascript", *assetsDir,
"third_party/jquery.timeago.js",
"third_party/typeahead.min.js",
"third_party/bootstrap/js/bootstrap.min.js",
"site.js"))
r.Add("/-/site.css").Get(dataHandler("site.css", "text/css", *assetsDir,
"third_party/bootstrap/css/bootstrap.min.css", "site.css"))
r.Add("/").GetFunc(serveHome)
r.Add("/-/about").GetFunc(serveAbout)
r.Add("/-/bot").GetFunc(serveBot)
r.Add("/-/opensearch.xml").GetFunc(serveOpenSearchDescription)
r.Add("/-/typeahead").GetFunc(serveTypeahead)
r.Add("/-/go").GetFunc(serveGoIndex)
r.Add("/-/subrepo").GetFunc(serveGoSubrepoIndex)
r.Add("/-/index").GetFunc(serveIndex)
r.Add("/-/refresh").PostFunc(serveRefresh)
r.Add("/-/static/<path:.*>").Get(staticConfig.DirectoryHandler("static"))
r.Add("/a/index").Get(web.RedirectHandler("/-/index", 301))
r.Add("/about").Get(web.RedirectHandler("/-/about", 301))
r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
r.Add("/robots.txt").Get(staticConfig.FileHandler("robots.txt"))
r.Add("/BingSiteAuth.xml").Get(staticConfig.FileHandler("BingSiteAuth.xml"))
r.Add("/C").Get(web.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301))
r.Add("/<path:.+>").GetFunc(servePackage)
h.Add("<:.*>", web.ErrorHandler(handleError, web.FormAndCookieHandler(1000, false, r)))
listener, err := net.Listen("tcp", *httpAddr)
if err != nil {
log.Fatal("Listen", err)
return
}
defer listener.Close()
s := &server.Server{Listener: listener, Handler: h} // add logger
err = s.Serve()
if err != nil {
log.Fatal("Server", err)
}
}
func prune(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
paths := make(map[string]bool)
err = db.Do(func(pi *database.PackageInfo) error {
pdoc := pi.PDoc
if pdoc.ProjectRoot == "" {
return nil
}
i := strings.LastIndex(pdoc.ImportPath, "/")
if i < 0 {
return nil
}
suffix := pdoc.ImportPath[i:]
imports := make(map[string]bool)
for _, p := range pdoc.Imports {
imports[p] = true
}
pathLists := [][]string{pdoc.TestImports, pdoc.XTestImports, pdoc.References}
if pdoc.ProjectRoot != pdoc.ImportPath {
if pdocRoot, _, _ := db.GetDoc(pdoc.ProjectRoot); pdocRoot != nil {
pathLists = append(pathLists, pdocRoot.References)
}
}
fork := ""
forkCheck:
for _, list := range pathLists {
for _, p := range list {
if p != pdoc.ImportPath && strings.HasSuffix(p, suffix) && !imports[p] {
pdocTest, _, _ := db.GetDoc(p)
if pdocTest != nil && pdocTest.Name == pdoc.Name && jaccardIndex(pdocTest, pdoc) > 0.75 {
fork = pdocTest.ImportPath
break forkCheck
}
}
}
}
if fork != "" {
log.Printf("%s is fork of %s", pdoc.ImportPath, fork)
if !*pruneDryRun {
for _, pkg := range pi.Pkgs {
if err := db.Delete(pkg.Path); err != nil {
log.Printf("Error deleting %s, %v", pkg.Path, err)
}
}
if err := db.Delete(pdoc.ImportPath); err != nil {
log.Printf("Error deleting %s, %v", pdoc.ImportPath, err)
}
}
} else {
keep := pi.Score > 0
if pdoc.IsCmd && pdoc.Synopsis != "" && len(pdoc.Doc) > len(pdoc.Synopsis) {
// Keep a command if there's actually some documentation.
keep = true
}
p := pdoc.ImportPath
for {
paths[p] = paths[p] || keep
if len(p) <= len(pdoc.ProjectRoot) {
break
} else if i := strings.LastIndex(p, "/"); i < 0 {
break
} else {
p = p[:i]
}
}
}
return nil
})
for p, keep := range paths {
if !keep {
log.Printf("%s has rank 0", p)
if !*pruneDryRun {
if err := db.Delete(p); err != nil {
log.Printf("Error deleting %s, %v", p, err)
}
}
}
}
if err != nil {
log.Fatal(err)
}
}