func stats(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
var packageSizes []itemSize
projectSizes := make(map[string]int)
err = db.Do(func(pi *database.PackageInfo) error {
packageSizes = append(packageSizes, itemSize{pi.PDoc.ImportPath, pi.Size})
projectSizes[pi.PDoc.ProjectRoot] += pi.Size
return nil
})
var sizes []itemSize
for path, size := range projectSizes {
sizes = append(sizes, itemSize{path, size})
}
sort.Sort(bySizeDesc(sizes))
for _, size := range sizes {
fmt.Printf("%6d %s\n", size.size, size.path)
}
sort.Sort(bySizeDesc(packageSizes))
for _, size := range packageSizes {
fmt.Printf("%6d %s\n", size.size, size.path)
}
}
func block(c *command) {
if len(c.flag.Args()) != 1 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
if err := db.Block(c.flag.Args()[0]); err != nil {
log.Fatal(err)
}
}
func popular(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
pkgs, err := db.PopularWithScores()
if err != nil {
log.Fatal(err)
}
for _, pkg := range pkgs {
fmt.Println(pkg.Path, pkg.Synopsis)
}
}
func stats(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
var packageSizes []itemSize
var truncatedPackages []string
projectSizes := make(map[string]int)
err = db.Do(func(pi *database.PackageInfo) error {
packageSizes = append(packageSizes, itemSize{pi.PDoc.ImportPath, pi.Size})
projectSizes[pi.PDoc.ProjectRoot] += pi.Size
if pi.PDoc.Truncated {
truncatedPackages = append(truncatedPackages, pi.PDoc.ImportPath)
}
return nil
})
var sizes []itemSize
for path, size := range projectSizes {
sizes = append(sizes, itemSize{path, size})
}
sort.Sort(bySizeDesc(sizes))
fmt.Println("PROJECT SIZES")
for _, size := range sizes {
fmt.Printf("%6d %s\n", size.size, size.path)
}
sort.Sort(bySizeDesc(packageSizes))
fmt.Println("PACKAGE SIZES")
for _, size := range packageSizes {
fmt.Printf("%6d %s\n", size.size, size.path)
}
sort.Sort(sort.StringSlice(truncatedPackages))
fmt.Println("TRUNCATED PACKAGES")
for _, p := range truncatedPackages {
fmt.Printf("%s\n", p)
}
}
func reindex(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
var n int
err = db.Do(func(pi *database.PackageInfo) error {
n++
fix(pi.PDoc)
return db.Put(pi.PDoc, time.Time{}, false)
})
if err != nil {
log.Fatal(err)
}
log.Printf("Updated %d documents", n)
}
func crawl(c *command) {
if len(c.flag.Args()) > 1 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
if len(c.flag.Args()) == 1 {
p, err := ioutil.ReadFile(c.flag.Args()[0])
if err != nil {
log.Fatal(err)
}
for _, p := range strings.Fields(string(p)) {
db.AddNewCrawl(p)
}
}
conn := db.Pool.Get()
defer conn.Close()
paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
if err != nil {
log.Fatal(err)
}
fmt.Println("NEW")
for _, path := range paths {
fmt.Println(path)
}
paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
if err != nil {
log.Fatal(err)
}
fmt.Println("BAD")
for _, path := range paths {
fmt.Println(path)
}
}
func dangle(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
m := make(map[string]int)
err = db.Do(func(pi *database.PackageInfo) error {
m[pi.PDoc.ImportPath] |= 1
for _, p := range pi.PDoc.Imports {
if gosrc.IsValidPath(p) {
m[p] |= 2
}
}
for _, p := range pi.PDoc.TestImports {
if gosrc.IsValidPath(p) {
m[p] |= 2
}
}
for _, p := range pi.PDoc.XTestImports {
if gosrc.IsValidPath(p) {
m[p] |= 2
}
}
return nil
})
for p, v := range m {
if v == 2 {
fmt.Println(p)
}
}
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
if *srcZip != "" {
r, err := zip.OpenReader(*srcZip)
if err != nil {
log.Fatal(err)
}
for _, f := range r.File {
if strings.HasPrefix(f.Name, "root/") {
srcFiles[f.Name[len("root/"):]] = f
}
}
}
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"dir.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"importers_robot.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"file.html", "common.html", "layout.html"},
{"index.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"tools.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"subrepo.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"dir.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
}); err != nil {
log.Fatal(err)
}
var err error
db, err = database.New()
if err != nil {
log.Fatalf("Error opening database: %v", err)
}
go runBackgroundTasks()
cssFiles := []string{"third_party/bootstrap/css/bootstrap.min.css", "site.css"}
staticServer := httputil.StaticServer{
Dir: *assetsDir,
MaxAge: time.Hour,
MIMETypes: map[string]string{
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
},
}
statusImageHandler = staticServer.FileHandler("status.png")
apiMux := http.NewServeMux()
apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
apiMux.Handle("/search", apiHandler(serveAPISearch))
apiMux.Handle("/packages", apiHandler(serveAPIPackages))
apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
apiMux.Handle("/imports/", apiHandler(serveAPIImports))
apiMux.Handle("/", apiHandler(serveAPIHome))
mux := http.NewServeMux()
mux.Handle("/-/site.js", staticServer.FilesHandler(
"third_party/jquery.timeago.js",
"third_party/typeahead.min.js",
"third_party/bootstrap/js/bootstrap.min.js",
"site.js"))
mux.Handle("/-/site.css", staticServer.FilesHandler(cssFiles...))
if *sidebarEnabled {
mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
}
mux.Handle("/-/about", handler(serveAbout))
mux.Handle("/-/bot", handler(serveBot))
mux.Handle("/-/go", handler(serveGoIndex))
mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
mux.Handle("/-/index", handler(serveIndex))
mux.Handle("/-/refresh", handler(serveRefresh))
mux.Handle("/a/index", http.RedirectHandler("/-/index", 301))
mux.Handle("/about", http.RedirectHandler("/-/about", 301))
mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301))
mux.Handle("/ajax.googleapis.com/", http.NotFoundHandler())
mux.Handle("/", handler(serveHome))
cacheBusters.Handler = mux
if err := http.ListenAndServe(*httpAddr, hostMux{{"api.", apiMux}, {"", mux}}); err != nil {
log.Fatal(err)
}
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
doc.SetDefaultGOOS(*defaultGOOS)
httpClient = newHTTPClient()
var (
gceLogName string
projID string
)
if metadata.OnGCE() {
acct, err := metadata.ProjectAttributeValue("ga-account")
if err != nil {
log.Printf("querying metadata for ga-account: %v", err)
} else {
gaAccount = acct
}
// Get the log name on GCE and setup context for creating a GCE log client.
if name, err := metadata.ProjectAttributeValue("gce-log-name"); err != nil {
log.Printf("querying metadata for gce-log-name: %v", err)
} else {
gceLogName = name
if id, err := metadata.ProjectID(); err != nil {
log.Printf("querying metadata for project ID: %v", err)
} else {
projID = id
}
}
} else {
gaAccount = os.Getenv("GA_ACCOUNT")
}
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"dir.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"importers_robot.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"tools.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"subrepo.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"dir.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
}); err != nil {
log.Fatal(err)
}
var err error
db, err = database.New()
if err != nil {
log.Fatalf("Error opening database: %v", err)
}
go runBackgroundTasks()
staticServer := httputil.StaticServer{
Dir: *assetsDir,
MaxAge: time.Hour,
MIMETypes: map[string]string{
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
},
}
statusImageHandlerPNG = staticServer.FileHandler("status.png")
statusImageHandlerSVG = staticServer.FileHandler("status.svg")
apiMux := http.NewServeMux()
apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
apiMux.Handle("/search", apiHandler(serveAPISearch))
apiMux.Handle("/packages", apiHandler(serveAPIPackages))
apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
apiMux.Handle("/imports/", apiHandler(serveAPIImports))
apiMux.Handle("/", apiHandler(serveAPIHome))
mux := http.NewServeMux()
mux.Handle("/-/site.js", staticServer.FilesHandler(
"third_party/jquery.timeago.js",
"site.js"))
mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
if *sidebarEnabled {
mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
}
mux.Handle("/-/", http.NotFoundHandler())
mux.Handle("/-/about", handler(serveAbout))
mux.Handle("/-/bot", handler(serveBot))
mux.Handle("/-/go", handler(serveGoIndex))
mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
mux.Handle("/-/refresh", handler(serveRefresh))
mux.Handle("/-/admin/reindex", http.HandlerFunc(runReindex))
mux.Handle("/-/admin/purgeindex", http.HandlerFunc(runPurgeIndex))
mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
mux.Handle("/code.jquery.com/", http.NotFoundHandler())
mux.Handle("/_ah/health", http.HandlerFunc(serveHealthCheck))
mux.Handle("/_ah/", http.NotFoundHandler())
mux.Handle("/", handler(serveHome))
cacheBusters.Handler = mux
var root http.Handler = rootHandler{
{"api.", apiMux},
{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
{"www.godoc.org", otherDomainHandler{"https", "godoc.org"}},
{"", mux},
}
if gceLogName != "" {
ctx := context.Background()
logc, err := logging.NewClient(ctx, projID)
if err != nil {
log.Fatalf("Failed to create cloud logging client: %v", err)
}
logger := logc.Logger(gceLogName)
if err := logc.Ping(ctx); err != nil {
log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
}
gceLogger = newGCELogger(logger)
}
http.Handle("/", root)
appengine.Main()
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
doc.SetDefaultGOOS(*defaultGOOS)
httpClient = newHTTPClient()
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"dir.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"importers_robot.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"index.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"tools.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"subrepo.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"dir.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
}); err != nil {
log.Fatal(err)
}
var err error
db, err = database.New()
if err != nil {
log.Fatalf("Error opening database: %v", err)
}
go runBackgroundTasks()
staticServer := httputil.StaticServer{
Dir: *assetsDir,
MaxAge: time.Hour,
MIMETypes: map[string]string{
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
},
}
statusImageHandlerPNG = staticServer.FileHandler("status.png")
statusImageHandlerSVG = staticServer.FileHandler("status.svg")
apiMux := http.NewServeMux()
apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
apiMux.Handle("/search", apiHandler(serveAPISearch))
apiMux.Handle("/packages", apiHandler(serveAPIPackages))
apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
apiMux.Handle("/imports/", apiHandler(serveAPIImports))
apiMux.Handle("/", apiHandler(serveAPIHome))
mux := http.NewServeMux()
mux.Handle("/-/site.js", staticServer.FilesHandler(
"third_party/jquery.timeago.js",
"site.js"))
mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
if *sidebarEnabled {
mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
}
mux.Handle("/-/about", handler(serveAbout))
mux.Handle("/-/bot", handler(serveBot))
mux.Handle("/-/go", handler(serveGoIndex))
mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
mux.Handle("/-/index", handler(serveIndex))
mux.Handle("/-/refresh", handler(serveRefresh))
mux.Handle("/a/index", http.RedirectHandler("/-/index", http.StatusMovedPermanently))
mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
mux.Handle("/code.jquery.com/", http.NotFoundHandler())
mux.Handle("/", handler(serveHome))
cacheBusters.Handler = mux
if err := http.ListenAndServe(*httpAddr, rootHandler{{"api.", apiMux}, {"", mux}}); err != nil {
log.Fatal(err)
}
}
func main() {
file := flag.String("file", "", "File containing the list of packages")
output := flag.String("output", "gddoscore.out", "Output file")
progress := flag.Bool("progress", false, "Show a progress bar")
flag.Parse()
var pkgs map[string]bool
var err error
if file != nil && *file != "" {
pkgs, err = readFromFile(*file)
} else {
pkgs, err = readFromStdin()
}
if err != nil {
fmt.Println(err)
return
}
db, err := database.New()
if err != nil {
fmt.Println("error connecting to database:", err)
return
}
o, err := os.OpenFile(*output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
fmt.Println("error creating output file:", err)
return
}
defer o.Close()
log.SetOutput(o)
log.Println("BEGIN")
log.Printf("%d packages will be analyzed\n", len(pkgs))
var progressBar *pb.ProgressBar
if progress != nil && *progress {
progressBar = pb.StartNew(len(pkgs))
}
db.Do(func(pkg *database.PackageInfo) error {
if _, ok := pkgs[pkg.PDoc.ImportPath]; !ok {
// we aren't analyzing this package
return nil
}
if pkg.Score == 0 {
log.Printf("package “%s” has no score", pkg.PDoc.ImportPath)
} else {
log.Printf("package “%s” has score", pkg.PDoc.ImportPath)
if progress != nil && !*progress {
fmt.Println(pkg.PDoc.ImportPath)
}
}
if progress != nil && *progress {
progressBar.Increment()
}
return nil
})
if progress != nil && *progress {
progressBar.Finish()
}
log.Println("END")
}
func main() {
clientID := flag.String("id", "", "Github client ID")
clientSecret := flag.String("secret", "", "Github client secret")
output := flag.String("output", "gddoexp.out", "Output file")
progress := flag.Bool("progress", false, "Show a progress bar")
flag.Parse()
var auth *gddoexp.GithubAuth
if (clientID != nil && *clientID != "") || (clientSecret != nil && *clientSecret != "") {
if *clientID == "" || *clientSecret == "" {
fmt.Println("to enable Gthub authentication, you need to inform the id and secret")
flag.PrintDefaults()
return
}
auth = &gddoexp.GithubAuth{
ID: *clientID,
Secret: *clientSecret,
}
}
// add cache to avoid repeated requests to Github
gddoexp.HTTPClient = &http.Client{
Transport: httpcache.NewTransport(
diskcache.New(path.Join(os.Getenv("HOME"), ".gddoexp")),
),
}
db, err := database.New()
if err != nil {
fmt.Println("error connecting to database:", err)
return
}
pkgs, err := db.AllPackages()
if err != nil {
fmt.Println("error retrieving all packages:", err)
return
}
file, err := os.OpenFile(*output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
fmt.Println("error creating output file:", err)
return
}
defer file.Close()
log.SetOutput(file)
log.Println("BEGIN")
log.Printf("%d packages will be analyzed", len(pkgs))
var progressBar *pb.ProgressBar
if progress != nil && *progress {
progressBar = pb.StartNew(len(pkgs))
}
var cache int
for response := range gddoexp.ShouldSuppressPackages(pkgs, db, auth) {
if progress != nil && *progress {
progressBar.Increment()
}
if response.Cache {
cache++
}
if response.Error != nil {
log.Println(response.Error)
} else if response.Suppress {
log.Printf("package “%s” should be suppressed\n", response.Package.Path)
if progress != nil && !*progress {
fmt.Println(response.Package.Path)
}
}
}
if progress != nil && *progress {
progressBar.Finish()
}
log.Println("Cache hits:", cache)
log.Println("END")
}