func init() { gob.Register(CrawlingEntry{}) doc.SetGithubCredentials("94446b37edb575accd8b", "15f55815f0515a3f6ad057aaffa9ea83dceb220b") doc.SetUserAgent("Go-Code-Search-Agent") }
func init() { if gcse.CrawlerGithubClientID != "" { log.Printf("Github clientid: %s", gcse.CrawlerGithubClientID) log.Printf("Github clientsecret: %s", gcse.CrawlerGithubClientSecret) doc.SetGithubCredentials(gcse.CrawlerGithubClientID, gcse.CrawlerGithubClientSecret) } doc.SetUserAgent("Go-Search(http://go-search.org/)") }
func TestGddo(t *testing.T) { doc.SetGithubCredentials("94446b37edb575accd8b", "15f55815f0515a3f6ad057aaffa9ea83dceb220b") doc.SetUserAgent("Go-Code-Search-Agent") pkg := "github.com/daviddengcn/gcse" p, err := CrawlPackage(http.DefaultClient, pkg, "") if err != nil { t.Error(err) } else { t.Logf("p: %+v", p.Exported) } // t.Error(nil) }
func init() { conf, _ := ljconf.Load("conf.json") serverAddr = conf.String("host", serverAddr) restSeconds = conf.Int("rest_seconds", restSeconds) proxyServer = conf.String("proxy", proxyServer) entriesPerLoop = conf.Int("entries_per_loop", entriesPerLoop) maxLoops = conf.Int("max_loops", maxLoops) doBlackPackages = conf.Bool("black_packages.enabled", doBlackPackages) if doBlackPackages { blackPkgFn = villa.Path(conf.String("black_packages.filename", blackPkgFn.S())) } doc.SetGithubCredentials("94446b37edb575accd8b", "15f55815f0515a3f6ad057aaffa9ea83dceb220b") doc.SetUserAgent("Go-Code-Search-Agent") }
func TestCrawlPackage(t *testing.T) { if CrawlerGithubClientID != "" { t.Logf("Github clientid: %s", CrawlerGithubClientID) t.Logf("Github clientsecret: %s", CrawlerGithubClientSecret) doc.SetGithubCredentials(CrawlerGithubClientID, CrawlerGithubClientSecret) } pkg := "github.com/daviddengcn/gcse" httpClient := GenHttpClient("") p, err := CrawlPackage(httpClient, pkg, "") if err != nil { if strings.Index(err.Error(), "403") == -1 { t.Error(err) } } else { assert.Equal(t, "pkg", p.Package, pkg) } // pkg = "git.gitorious.org/go-pkg/epubgo.git" // p, err = CrawlPackage(httpClient, pkg, "") // if err != nil { // if strings.Index(err.Error(), "403") == -1 { // t.Error(err) // } // } else { // assert.Equal(t, "pkg", p.Package, pkg) // } pkg = "thezombie.net/libgojira" p, err = CrawlPackage(httpClient, pkg, "") if err != nil { if !IsBadPackage(err) { t.Errorf("%s should be an invalid package", pkg) } } else { t.Errorf("%s should be an invalid package", pkg) } }
func init() { doc.SetGithubCredentials("94446b37edb575accd8b", "15f55815f0515a3f6ad057aaffa9ea83dceb220b") doc.SetUserAgent("Go-Code-Search-Engine") }