func stats(c *command) { if len(c.flag.Args()) != 0 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } var packageSizes []itemSize projectSizes := make(map[string]int) err = db.Do(func(pi *database.PackageInfo) error { packageSizes = append(packageSizes, itemSize{pi.PDoc.ImportPath, pi.Size}) projectSizes[pi.PDoc.ProjectRoot] += pi.Size return nil }) var sizes []itemSize for path, size := range projectSizes { sizes = append(sizes, itemSize{path, size}) } sort.Sort(bySizeDesc(sizes)) for _, size := range sizes { fmt.Printf("%6d %s\n", size.size, size.path) } sort.Sort(bySizeDesc(packageSizes)) for _, size := range packageSizes { fmt.Printf("%6d %s\n", size.size, size.path) } }
func block(c *command) { if len(c.flag.Args()) != 1 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } if err := db.Block(c.flag.Args()[0]); err != nil { log.Fatal(err) } }
func popular(c *command) { if len(c.flag.Args()) != 0 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } pkgs, err := db.PopularWithScores() if err != nil { log.Fatal(err) } for _, pkg := range pkgs { fmt.Println(pkg.Path, pkg.Synopsis) } }
func stats(c *command) { if len(c.flag.Args()) != 0 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } var packageSizes []itemSize var truncatedPackages []string projectSizes := make(map[string]int) err = db.Do(func(pi *database.PackageInfo) error { packageSizes = append(packageSizes, itemSize{pi.PDoc.ImportPath, pi.Size}) projectSizes[pi.PDoc.ProjectRoot] += pi.Size if pi.PDoc.Truncated { truncatedPackages = append(truncatedPackages, pi.PDoc.ImportPath) } return nil }) var sizes []itemSize for path, size := range projectSizes { sizes = append(sizes, itemSize{path, size}) } sort.Sort(bySizeDesc(sizes)) fmt.Println("PROJECT SIZES") for _, size := range sizes { fmt.Printf("%6d %s\n", size.size, size.path) } sort.Sort(bySizeDesc(packageSizes)) fmt.Println("PACKAGE SIZES") for _, size := range packageSizes { fmt.Printf("%6d %s\n", size.size, size.path) } sort.Sort(sort.StringSlice(truncatedPackages)) fmt.Println("TRUNCATED PACKAGES") for _, p := range truncatedPackages { fmt.Printf("%s\n", p) } }
func reindex(c *command) { if len(c.flag.Args()) != 0 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } var n int err = db.Do(func(pi *database.PackageInfo) error { n++ fix(pi.PDoc) return db.Put(pi.PDoc, time.Time{}, false) }) if err != nil { log.Fatal(err) } log.Printf("Updated %d documents", n) }
func crawl(c *command) { if len(c.flag.Args()) > 1 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } if len(c.flag.Args()) == 1 { p, err := ioutil.ReadFile(c.flag.Args()[0]) if err != nil { log.Fatal(err) } for _, p := range strings.Fields(string(p)) { db.AddNewCrawl(p) } } conn := db.Pool.Get() defer conn.Close() paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl")) if err != nil { log.Fatal(err) } fmt.Println("NEW") for _, path := range paths { fmt.Println(path) } paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl")) if err != nil { log.Fatal(err) } fmt.Println("BAD") for _, path := range paths { fmt.Println(path) } }
func dangle(c *command) { if len(c.flag.Args()) != 0 { c.printUsage() os.Exit(1) } db, err := database.New() if err != nil { log.Fatal(err) } m := make(map[string]int) err = db.Do(func(pi *database.PackageInfo) error { m[pi.PDoc.ImportPath] |= 1 for _, p := range pi.PDoc.Imports { if gosrc.IsValidPath(p) { m[p] |= 2 } } for _, p := range pi.PDoc.TestImports { if gosrc.IsValidPath(p) { m[p] |= 2 } } for _, p := range pi.PDoc.XTestImports { if gosrc.IsValidPath(p) { m[p] |= 2 } } return nil }) for p, v := range m { if v == 2 { fmt.Println(p) } } }
func main() { flag.Parse() log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " ")) if *srcZip != "" { r, err := zip.OpenReader(*srcZip) if err != nil { log.Fatal(err) } for _, f := range r.File { if strings.HasPrefix(f.Name, "root/") { srcFiles[f.Name[len("root/"):]] = f } } } if err := parseHTMLTemplates([][]string{ {"about.html", "common.html", "layout.html"}, {"bot.html", "common.html", "layout.html"}, {"cmd.html", "common.html", "layout.html"}, {"dir.html", "common.html", "layout.html"}, {"home.html", "common.html", "layout.html"}, {"importers.html", "common.html", "layout.html"}, {"importers_robot.html", "common.html", "layout.html"}, {"imports.html", "common.html", "layout.html"}, {"file.html", "common.html", "layout.html"}, {"index.html", "common.html", "layout.html"}, {"notfound.html", "common.html", "layout.html"}, {"pkg.html", "common.html", "layout.html"}, {"results.html", "common.html", "layout.html"}, {"tools.html", "common.html", "layout.html"}, {"std.html", "common.html", "layout.html"}, {"subrepo.html", "common.html", "layout.html"}, {"graph.html", "common.html"}, }); err != nil { log.Fatal(err) } if err := parseTextTemplates([][]string{ {"cmd.txt", "common.txt"}, {"dir.txt", "common.txt"}, {"home.txt", "common.txt"}, {"notfound.txt", "common.txt"}, {"pkg.txt", "common.txt"}, {"results.txt", "common.txt"}, }); err != nil { log.Fatal(err) } var err error db, err = database.New() if err != nil { log.Fatalf("Error opening database: %v", err) } go runBackgroundTasks() cssFiles := []string{"third_party/bootstrap/css/bootstrap.min.css", "site.css"} staticServer := httputil.StaticServer{ Dir: *assetsDir, MaxAge: time.Hour, MIMETypes: map[string]string{ ".css": "text/css; charset=utf-8", ".js": "text/javascript; charset=utf-8", }, } statusImageHandler = staticServer.FileHandler("status.png") apiMux := http.NewServeMux() apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt")) apiMux.Handle("/search", apiHandler(serveAPISearch)) apiMux.Handle("/packages", apiHandler(serveAPIPackages)) apiMux.Handle("/importers/", apiHandler(serveAPIImporters)) apiMux.Handle("/imports/", apiHandler(serveAPIImports)) apiMux.Handle("/", apiHandler(serveAPIHome)) mux := http.NewServeMux() mux.Handle("/-/site.js", staticServer.FilesHandler( "third_party/jquery.timeago.js", "third_party/typeahead.min.js", "third_party/bootstrap/js/bootstrap.min.js", "site.js")) mux.Handle("/-/site.css", staticServer.FilesHandler(cssFiles...)) if *sidebarEnabled { mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css")) } mux.Handle("/-/about", handler(serveAbout)) mux.Handle("/-/bot", handler(serveBot)) mux.Handle("/-/go", handler(serveGoIndex)) mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex)) mux.Handle("/-/index", handler(serveIndex)) mux.Handle("/-/refresh", handler(serveRefresh)) mux.Handle("/a/index", http.RedirectHandler("/-/index", 301)) mux.Handle("/about", http.RedirectHandler("/-/about", 301)) mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt")) mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml")) mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301)) mux.Handle("/ajax.googleapis.com/", http.NotFoundHandler()) mux.Handle("/", handler(serveHome)) cacheBusters.Handler = mux if err := http.ListenAndServe(*httpAddr, hostMux{{"api.", apiMux}, {"", mux}}); err != nil { log.Fatal(err) } }
func main() { flag.Parse() log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " ")) doc.SetDefaultGOOS(*defaultGOOS) httpClient = newHTTPClient() var ( gceLogName string projID string ) if metadata.OnGCE() { acct, err := metadata.ProjectAttributeValue("ga-account") if err != nil { log.Printf("querying metadata for ga-account: %v", err) } else { gaAccount = acct } // Get the log name on GCE and setup context for creating a GCE log client. if name, err := metadata.ProjectAttributeValue("gce-log-name"); err != nil { log.Printf("querying metadata for gce-log-name: %v", err) } else { gceLogName = name if id, err := metadata.ProjectID(); err != nil { log.Printf("querying metadata for project ID: %v", err) } else { projID = id } } } else { gaAccount = os.Getenv("GA_ACCOUNT") } if err := parseHTMLTemplates([][]string{ {"about.html", "common.html", "layout.html"}, {"bot.html", "common.html", "layout.html"}, {"cmd.html", "common.html", "layout.html"}, {"dir.html", "common.html", "layout.html"}, {"home.html", "common.html", "layout.html"}, {"importers.html", "common.html", "layout.html"}, {"importers_robot.html", "common.html", "layout.html"}, {"imports.html", "common.html", "layout.html"}, {"notfound.html", "common.html", "layout.html"}, {"pkg.html", "common.html", "layout.html"}, {"results.html", "common.html", "layout.html"}, {"tools.html", "common.html", "layout.html"}, {"std.html", "common.html", "layout.html"}, {"subrepo.html", "common.html", "layout.html"}, {"graph.html", "common.html"}, }); err != nil { log.Fatal(err) } if err := parseTextTemplates([][]string{ {"cmd.txt", "common.txt"}, {"dir.txt", "common.txt"}, {"home.txt", "common.txt"}, {"notfound.txt", "common.txt"}, {"pkg.txt", "common.txt"}, {"results.txt", "common.txt"}, }); err != nil { log.Fatal(err) } var err error db, err = database.New() if err != nil { log.Fatalf("Error opening database: %v", err) } go runBackgroundTasks() staticServer := httputil.StaticServer{ Dir: *assetsDir, MaxAge: time.Hour, MIMETypes: map[string]string{ ".css": "text/css; charset=utf-8", ".js": "text/javascript; charset=utf-8", }, } statusImageHandlerPNG = staticServer.FileHandler("status.png") statusImageHandlerSVG = staticServer.FileHandler("status.svg") apiMux := http.NewServeMux() apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt")) apiMux.Handle("/search", apiHandler(serveAPISearch)) apiMux.Handle("/packages", apiHandler(serveAPIPackages)) apiMux.Handle("/importers/", apiHandler(serveAPIImporters)) apiMux.Handle("/imports/", apiHandler(serveAPIImports)) apiMux.Handle("/", apiHandler(serveAPIHome)) mux := http.NewServeMux() mux.Handle("/-/site.js", staticServer.FilesHandler( "third_party/jquery.timeago.js", "site.js")) mux.Handle("/-/site.css", staticServer.FilesHandler("site.css")) mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css")) mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js")) mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js")) if *sidebarEnabled { mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css")) } mux.Handle("/-/", http.NotFoundHandler()) mux.Handle("/-/about", handler(serveAbout)) mux.Handle("/-/bot", handler(serveBot)) mux.Handle("/-/go", handler(serveGoIndex)) mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex)) mux.Handle("/-/refresh", handler(serveRefresh)) mux.Handle("/-/admin/reindex", http.HandlerFunc(runReindex)) mux.Handle("/-/admin/purgeindex", http.HandlerFunc(runPurgeIndex)) mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently)) mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt")) mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml")) mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently)) mux.Handle("/code.jquery.com/", http.NotFoundHandler()) mux.Handle("/_ah/health", http.HandlerFunc(serveHealthCheck)) mux.Handle("/_ah/", http.NotFoundHandler()) mux.Handle("/", handler(serveHome)) cacheBusters.Handler = mux var root http.Handler = rootHandler{ {"api.", apiMux}, {"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}}, {"www.godoc.org", otherDomainHandler{"https", "godoc.org"}}, {"", mux}, } if gceLogName != "" { ctx := context.Background() logc, err := logging.NewClient(ctx, projID) if err != nil { log.Fatalf("Failed to create cloud logging client: %v", err) } logger := logc.Logger(gceLogName) if err := logc.Ping(ctx); err != nil { log.Fatalf("Failed to ping Google Cloud Logging: %v", err) } gceLogger = newGCELogger(logger) } http.Handle("/", root) appengine.Main() }
func main() { flag.Parse() log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " ")) doc.SetDefaultGOOS(*defaultGOOS) httpClient = newHTTPClient() if err := parseHTMLTemplates([][]string{ {"about.html", "common.html", "layout.html"}, {"bot.html", "common.html", "layout.html"}, {"cmd.html", "common.html", "layout.html"}, {"dir.html", "common.html", "layout.html"}, {"home.html", "common.html", "layout.html"}, {"importers.html", "common.html", "layout.html"}, {"importers_robot.html", "common.html", "layout.html"}, {"imports.html", "common.html", "layout.html"}, {"index.html", "common.html", "layout.html"}, {"notfound.html", "common.html", "layout.html"}, {"pkg.html", "common.html", "layout.html"}, {"results.html", "common.html", "layout.html"}, {"tools.html", "common.html", "layout.html"}, {"std.html", "common.html", "layout.html"}, {"subrepo.html", "common.html", "layout.html"}, {"graph.html", "common.html"}, }); err != nil { log.Fatal(err) } if err := parseTextTemplates([][]string{ {"cmd.txt", "common.txt"}, {"dir.txt", "common.txt"}, {"home.txt", "common.txt"}, {"notfound.txt", "common.txt"}, {"pkg.txt", "common.txt"}, {"results.txt", "common.txt"}, }); err != nil { log.Fatal(err) } var err error db, err = database.New() if err != nil { log.Fatalf("Error opening database: %v", err) } go runBackgroundTasks() staticServer := httputil.StaticServer{ Dir: *assetsDir, MaxAge: time.Hour, MIMETypes: map[string]string{ ".css": "text/css; charset=utf-8", ".js": "text/javascript; charset=utf-8", }, } statusImageHandlerPNG = staticServer.FileHandler("status.png") statusImageHandlerSVG = staticServer.FileHandler("status.svg") apiMux := http.NewServeMux() apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt")) apiMux.Handle("/search", apiHandler(serveAPISearch)) apiMux.Handle("/packages", apiHandler(serveAPIPackages)) apiMux.Handle("/importers/", apiHandler(serveAPIImporters)) apiMux.Handle("/imports/", apiHandler(serveAPIImports)) apiMux.Handle("/", apiHandler(serveAPIHome)) mux := http.NewServeMux() mux.Handle("/-/site.js", staticServer.FilesHandler( "third_party/jquery.timeago.js", "site.js")) mux.Handle("/-/site.css", staticServer.FilesHandler("site.css")) if *sidebarEnabled { mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css")) } mux.Handle("/-/about", handler(serveAbout)) mux.Handle("/-/bot", handler(serveBot)) mux.Handle("/-/go", handler(serveGoIndex)) mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex)) mux.Handle("/-/index", handler(serveIndex)) mux.Handle("/-/refresh", handler(serveRefresh)) mux.Handle("/a/index", http.RedirectHandler("/-/index", http.StatusMovedPermanently)) mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently)) mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt")) mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml")) mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently)) mux.Handle("/code.jquery.com/", http.NotFoundHandler()) mux.Handle("/", handler(serveHome)) cacheBusters.Handler = mux if err := http.ListenAndServe(*httpAddr, rootHandler{{"api.", apiMux}, {"", mux}}); err != nil { log.Fatal(err) } }
func main() { file := flag.String("file", "", "File containing the list of packages") output := flag.String("output", "gddoscore.out", "Output file") progress := flag.Bool("progress", false, "Show a progress bar") flag.Parse() var pkgs map[string]bool var err error if file != nil && *file != "" { pkgs, err = readFromFile(*file) } else { pkgs, err = readFromStdin() } if err != nil { fmt.Println(err) return } db, err := database.New() if err != nil { fmt.Println("error connecting to database:", err) return } o, err := os.OpenFile(*output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { fmt.Println("error creating output file:", err) return } defer o.Close() log.SetOutput(o) log.Println("BEGIN") log.Printf("%d packages will be analyzed\n", len(pkgs)) var progressBar *pb.ProgressBar if progress != nil && *progress { progressBar = pb.StartNew(len(pkgs)) } db.Do(func(pkg *database.PackageInfo) error { if _, ok := pkgs[pkg.PDoc.ImportPath]; !ok { // we aren't analyzing this package return nil } if pkg.Score == 0 { log.Printf("package “%s” has no score", pkg.PDoc.ImportPath) } else { log.Printf("package “%s” has score", pkg.PDoc.ImportPath) if progress != nil && !*progress { fmt.Println(pkg.PDoc.ImportPath) } } if progress != nil && *progress { progressBar.Increment() } return nil }) if progress != nil && *progress { progressBar.Finish() } log.Println("END") }
func main() { clientID := flag.String("id", "", "Github client ID") clientSecret := flag.String("secret", "", "Github client secret") output := flag.String("output", "gddoexp.out", "Output file") progress := flag.Bool("progress", false, "Show a progress bar") flag.Parse() var auth *gddoexp.GithubAuth if (clientID != nil && *clientID != "") || (clientSecret != nil && *clientSecret != "") { if *clientID == "" || *clientSecret == "" { fmt.Println("to enable Gthub authentication, you need to inform the id and secret") flag.PrintDefaults() return } auth = &gddoexp.GithubAuth{ ID: *clientID, Secret: *clientSecret, } } // add cache to avoid repeated requests to Github gddoexp.HTTPClient = &http.Client{ Transport: httpcache.NewTransport( diskcache.New(path.Join(os.Getenv("HOME"), ".gddoexp")), ), } db, err := database.New() if err != nil { fmt.Println("error connecting to database:", err) return } pkgs, err := db.AllPackages() if err != nil { fmt.Println("error retrieving all packages:", err) return } file, err := os.OpenFile(*output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { fmt.Println("error creating output file:", err) return } defer file.Close() log.SetOutput(file) log.Println("BEGIN") log.Printf("%d packages will be analyzed", len(pkgs)) var progressBar *pb.ProgressBar if progress != nil && *progress { progressBar = pb.StartNew(len(pkgs)) } var cache int for response := range gddoexp.ShouldSuppressPackages(pkgs, db, auth) { if progress != nil && *progress { progressBar.Increment() } if response.Cache { cache++ } if response.Error != nil { log.Println(response.Error) } else if response.Suppress { log.Printf("package “%s” should be suppressed\n", response.Package.Path) if progress != nil && !*progress { fmt.Println(response.Package.Path) } } } if progress != nil && *progress { progressBar.Finish() } log.Println("Cache hits:", cache) log.Println("END") }