Beispiel #1
0
func crawl(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	conn := db.Pool.Get()
	defer conn.Close()

	paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("NEW")
	for _, path := range paths {
		fmt.Println(path)
	}

	paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("BAD")
	for _, path := range paths {
		fmt.Println(path)
	}
}
Beispiel #2
0
func stats(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	_, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("DONE")
}
Beispiel #3
0
func block(c *command) {
	if len(c.flag.Args()) != 1 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	if err := db.Block(c.flag.Args()[0]); err != nil {
		log.Fatal(err)
	}
}
Beispiel #4
0
func popular(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	pkgs, err := db.PopularWithScores()
	if err != nil {
		log.Fatal(err)
	}
	for _, pkg := range pkgs {
		fmt.Println(pkg.Path, pkg.Synopsis)
	}
}
Beispiel #5
0
func reindex(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	var n int
	err = db.Do(func(pi *database.PackageInfo) error {
		n += 1
		fix(pi.PDoc)
		return db.Put(pi.PDoc, time.Time{})
	})
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("Updated %d documents", n)
}
Beispiel #6
0
func dangle(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	m := make(map[string]int)
	err = db.Do(func(pi *database.PackageInfo) error {
		m[pi.PDoc.ImportPath] |= 1
		for _, p := range pi.PDoc.Imports {
			if doc.IsValidPath(p) {
				m[p] |= 2
			}
		}
		for _, p := range pi.PDoc.TestImports {
			if doc.IsValidPath(p) {
				m[p] |= 2
			}
		}
		for _, p := range pi.PDoc.XTestImports {
			if doc.IsValidPath(p) {
				m[p] |= 2
			}
		}
		return nil
	})

	for p, v := range m {
		if v == 2 {
			fmt.Println(p)
		}
	}
}
Beispiel #7
0
func main() {
	flag.Parse()
	log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
	if err := readSecrets(); err != nil {
		log.Fatal(err)
	}

	if err := parseHTMLTemplates([][]string{
		{"about.html", "common.html", "layout.html"},
		{"bot.html", "common.html", "layout.html"},
		{"cmd.html", "common.html", "layout.html"},
		{"home.html", "common.html", "layout.html"},
		{"importers.html", "common.html", "layout.html"},
		{"imports.html", "common.html", "layout.html"},
		{"interface.html", "common.html", "layout.html"},
		{"index.html", "common.html", "layout.html"},
		{"notfound.html", "common.html", "layout.html"},
		{"pkg.html", "common.html", "layout.html"},
		{"results.html", "common.html", "layout.html"},
		{"std.html", "common.html", "layout.html"},
		{"graph.html", "common.html"},
	}); err != nil {
		log.Fatal(err)
	}

	if err := parseTextTemplates([][]string{
		{"cmd.txt", "common.txt"},
		{"home.txt", "common.txt"},
		{"notfound.txt", "common.txt"},
		{"pkg.txt", "common.txt"},
		{"results.txt", "common.txt"},
		{"opensearch.xml"},
	}); err != nil {
		log.Fatal(err)
	}

	if err := parsePresentTemplates([][]string{
		{".article", "article.tmpl", "action.tmpl"},
		{".slide", "slides.tmpl", "action.tmpl"},
	}); err != nil {
		log.Fatal(err)
	}

	present.PlayEnabled = true

	var err error
	db, err = database.New()
	if err != nil {
		log.Fatal(err)
	}

	if *crawlInterval > 0 {
		go crawl(*crawlInterval)
	}

	if *githubInterval > 0 {
		go crawlGithubUpdates(*githubInterval)
	}

	playScript, err := readPlayScript(*presentDir)
	if err != nil {
		log.Fatal(err)
	}

	staticConfig := &web.StaticConfig{
		Header:      web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
		Directory:   *assetsDir,
		GzDirectory: *gzAssetsDir,
	}
	presentStaticConfig := &web.StaticConfig{
		Header:    web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
		Directory: *presentDir,
	}

	h := web.NewHostRouter()

	r := web.NewRouter()
	r.Add("/").GetFunc(servePresentHome)
	r.Add("/compile").PostFunc(serveCompile)
	r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
	r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
	r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
	r.Add("/play.js").Get(web.DataHandler(playScript, web.Header{web.HeaderContentType: {"text/javascript"}}))
	r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
	r.Add("/static/<path:.*>").Get(presentStaticConfig.DirectoryHandler("static"))
	r.Add("/<path:.+>").GetFunc(servePresentation)

	h.Add("talks.<:.*>", web.ErrorHandler(handlePresentError, web.FormAndCookieHandler(6000, false, r)))

	r = web.NewRouter()
	r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
	r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
	r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
	r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
	r.Add("/search").GetFunc(serveAPISearch)
	r.Add("/packages").GetFunc(serveAPIPackages)

	h.Add("api.<:.*>", web.ErrorHandler(handleAPIError, web.FormAndCookieHandler(6000, false, r)))

	r = web.NewRouter()
	r.Add("/").GetFunc(serveHome)
	r.Add("/-/about").GetFunc(serveAbout)
	r.Add("/-/bot").GetFunc(serveBot)
	r.Add("/-/opensearch.xml").GetFunc(serveOpenSearchDescription)
	r.Add("/-/typeahead").GetFunc(serveTypeahead)
	r.Add("/-/go").GetFunc(serveGoIndex)
	r.Add("/-/index").GetFunc(serveIndex)
	r.Add("/-/refresh").PostFunc(serveRefresh)
	r.Add("/-/static/<path:.*>").Get(staticConfig.DirectoryHandler("static"))
	r.Add("/a/index").Get(web.RedirectHandler("/-/index", 301))
	r.Add("/about").Get(web.RedirectHandler("/-/about", 301))
	r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
	r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
	r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
	r.Add("/robots.txt").Get(staticConfig.FileHandler("robots.txt"))
	r.Add("/BingSiteAuth.xml").Get(staticConfig.FileHandler("BingSiteAuth.xml"))
	r.Add("/C").Get(web.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301))
	r.Add("/<path:.+>").GetFunc(servePackage)

	h.Add("<:.*>", web.ErrorHandler(handleError, web.FormAndCookieHandler(1000, false, r)))

	listener, err := net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatal("Listen", err)
		return
	}
	defer listener.Close()
	s := &server.Server{Listener: listener, Handler: h} // add logger
	err = s.Serve()
	if err != nil {
		log.Fatal("Server", err)
	}
}
Beispiel #8
0
func main() {
	flag.Parse()
	log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
	if err := readSecrets(); err != nil {
		log.Fatal(err)
	}

	if *srcZip != "" {
		r, err := zip.OpenReader(*srcZip)
		if err != nil {
			log.Fatal(err)
		}
		for _, f := range r.File {
			if strings.HasPrefix(f.Name, "root/") {
				srcFiles[f.Name[len("root/"):]] = f
			}
		}
	}

	if err := parseHTMLTemplates([][]string{
		{"about.html", "common.html", "layout.html"},
		{"bot.html", "common.html", "layout.html"},
		{"cmd.html", "common.html", "layout.html"},
		{"dir.html", "common.html", "layout.html"},
		{"home.html", "common.html", "layout.html"},
		{"importers.html", "common.html", "layout.html"},
		{"imports.html", "common.html", "layout.html"},
		{"file.html", "common.html", "layout.html"},
		{"index.html", "common.html", "layout.html"},
		{"notfound.html", "common.html", "layout.html"},
		{"pkg.html", "common.html", "layout.html"},
		{"results.html", "common.html", "layout.html"},
		{"std.html", "common.html", "layout.html"},
		{"subrepo.html", "common.html", "layout.html"},
		{"graph.html", "common.html"},
	}); err != nil {
		log.Fatal(err)
	}

	if err := parseTextTemplates([][]string{
		{"cmd.txt", "common.txt"},
		{"dir.txt", "common.txt"},
		{"home.txt", "common.txt"},
		{"notfound.txt", "common.txt"},
		{"pkg.txt", "common.txt"},
		{"results.txt", "common.txt"},
		{"opensearch.xml"},
	}); err != nil {
		log.Fatal(err)
	}

	if err := parsePresentTemplates([][]string{
		{".article", "article.tmpl", "action.tmpl"},
		{".slide", "slides.tmpl", "action.tmpl"},
	}); err != nil {
		log.Fatal(err)
	}

	present.PlayEnabled = true

	var err error
	db, err = database.New()
	if err != nil {
		log.Fatal(err)
	}

	go runBackgroundTasks()

	staticConfig := &web.StaticConfig{
		Header:      web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
		Directory:   *assetsDir,
		GzDirectory: *gzAssetsDir,
	}
	presentStaticConfig := &web.StaticConfig{
		Header:    web.Header{web.HeaderCacheControl: {"public, max-age=3600"}},
		Directory: *presentDir,
	}

	h := web.NewHostRouter()

	r := web.NewRouter()
	r.Add("/").GetFunc(servePresentHome)
	r.Add("/compile").PostFunc(serveCompile)
	r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
	r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
	r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
	r.Add("/play.js").Get(dataHandler("play.js", "text/javascript", *presentDir, "js/jquery.js", "js/playground.js", "js/play.js"))
	r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
	r.Add("/static/<path:.*>").Get(presentStaticConfig.DirectoryHandler("static"))
	if *redirGoTalks {
		r.Add("/code.google.com/p/go.talks/<path:.+>").GetFunc(func(resp web.Response, req *web.Request) error {
			return web.Redirect(resp, req, "http://talks.golang.org/"+req.RouteVars["path"], 301, nil)
		})
	}
	r.Add("/<path:.+>").GetFunc(servePresentation)

	h.Add("talks.<:.*>", web.ErrorHandler(handlePresentError, web.FormAndCookieHandler(6000, false, r)))

	r = web.NewRouter()
	r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
	r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
	r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
	r.Add("/robots.txt").Get(staticConfig.FileHandler("presentRobots.txt"))
	r.Add("/search").GetFunc(serveAPISearch)
	r.Add("/packages").GetFunc(serveAPIPackages)
	r.Add("/importers/<path:.+>").GetFunc(serveAPIImporters)

	h.Add("api.<:.*>", web.ErrorHandler(handleAPIError, web.FormAndCookieHandler(6000, false, r)))

	r = web.NewRouter()
	r.Add("/-/site.js").Get(dataHandler("site.js", "text/javascript", *assetsDir,
		"third_party/jquery.timeago.js",
		"third_party/typeahead.min.js",
		"third_party/bootstrap/js/bootstrap.min.js",
		"site.js"))
	r.Add("/-/site.css").Get(dataHandler("site.css", "text/css", *assetsDir,
		"third_party/bootstrap/css/bootstrap.min.css", "site.css"))
	r.Add("/").GetFunc(serveHome)
	r.Add("/-/about").GetFunc(serveAbout)
	r.Add("/-/bot").GetFunc(serveBot)
	r.Add("/-/opensearch.xml").GetFunc(serveOpenSearchDescription)
	r.Add("/-/typeahead").GetFunc(serveTypeahead)
	r.Add("/-/go").GetFunc(serveGoIndex)
	r.Add("/-/subrepo").GetFunc(serveGoSubrepoIndex)
	r.Add("/-/index").GetFunc(serveIndex)
	r.Add("/-/refresh").PostFunc(serveRefresh)
	r.Add("/-/static/<path:.*>").Get(staticConfig.DirectoryHandler("static"))
	r.Add("/a/index").Get(web.RedirectHandler("/-/index", 301))
	r.Add("/about").Get(web.RedirectHandler("/-/about", 301))
	r.Add("/favicon.ico").Get(staticConfig.FileHandler("favicon.ico"))
	r.Add("/google3d2f3cd4cc2bb44b.html").Get(staticConfig.FileHandler("google3d2f3cd4cc2bb44b.html"))
	r.Add("/humans.txt").Get(staticConfig.FileHandler("humans.txt"))
	r.Add("/robots.txt").Get(staticConfig.FileHandler("robots.txt"))
	r.Add("/BingSiteAuth.xml").Get(staticConfig.FileHandler("BingSiteAuth.xml"))
	r.Add("/C").Get(web.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301))
	r.Add("/<path:.+>").GetFunc(servePackage)

	h.Add("<:.*>", web.ErrorHandler(handleError, web.FormAndCookieHandler(1000, false, r)))

	listener, err := net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatal("Listen", err)
		return
	}
	defer listener.Close()
	s := &server.Server{Listener: listener, Handler: h} // add logger
	err = s.Serve()
	if err != nil {
		log.Fatal("Server", err)
	}
}
Beispiel #9
0
func prune(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}

	paths := make(map[string]bool)

	err = db.Do(func(pi *database.PackageInfo) error {
		pdoc := pi.PDoc
		if pdoc.ProjectRoot == "" {
			return nil
		}

		i := strings.LastIndex(pdoc.ImportPath, "/")
		if i < 0 {
			return nil
		}
		suffix := pdoc.ImportPath[i:]

		imports := make(map[string]bool)
		for _, p := range pdoc.Imports {
			imports[p] = true
		}

		pathLists := [][]string{pdoc.TestImports, pdoc.XTestImports, pdoc.References}
		if pdoc.ProjectRoot != pdoc.ImportPath {
			if pdocRoot, _, _ := db.GetDoc(pdoc.ProjectRoot); pdocRoot != nil {
				pathLists = append(pathLists, pdocRoot.References)
			}
		}

		fork := ""

	forkCheck:
		for _, list := range pathLists {
			for _, p := range list {
				if p != pdoc.ImportPath && strings.HasSuffix(p, suffix) && !imports[p] {
					pdocTest, _, _ := db.GetDoc(p)
					if pdocTest != nil && pdocTest.Name == pdoc.Name && jaccardIndex(pdocTest, pdoc) > 0.75 {
						fork = pdocTest.ImportPath
						break forkCheck
					}
				}
			}
		}

		if fork != "" {
			log.Printf("%s is fork of %s", pdoc.ImportPath, fork)
			if !*pruneDryRun {
				for _, pkg := range pi.Pkgs {
					if err := db.Delete(pkg.Path); err != nil {
						log.Printf("Error deleting %s, %v", pkg.Path, err)
					}
				}
				if err := db.Delete(pdoc.ImportPath); err != nil {
					log.Printf("Error deleting %s, %v", pdoc.ImportPath, err)
				}
			}
		} else {
			keep := pi.Score > 0
			if pdoc.IsCmd && pdoc.Synopsis != "" && len(pdoc.Doc) > len(pdoc.Synopsis) {
				// Keep a command if there's actually some documentation.
				keep = true
			}
			p := pdoc.ImportPath
			for {
				paths[p] = paths[p] || keep
				if len(p) <= len(pdoc.ProjectRoot) {
					break
				} else if i := strings.LastIndex(p, "/"); i < 0 {
					break
				} else {
					p = p[:i]
				}
			}
		}
		return nil
	})

	for p, keep := range paths {
		if !keep {
			log.Printf("%s has rank 0", p)
			if !*pruneDryRun {
				if err := db.Delete(p); err != nil {
					log.Printf("Error deleting %s, %v", p, err)
				}
			}
		}
	}

	if err != nil {
		log.Fatal(err)
	}
}