func init() {
gob.Register(CrawlingEntry{})
doc.SetGithubCredentials("94446b37edb575accd8b",
"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
doc.SetUserAgent("Go-Code-Search-Agent")
}
func init() {
if gcse.CrawlerGithubClientID != "" {
log.Printf("Github clientid: %s", gcse.CrawlerGithubClientID)
log.Printf("Github clientsecret: %s", gcse.CrawlerGithubClientSecret)
doc.SetGithubCredentials(gcse.CrawlerGithubClientID, gcse.CrawlerGithubClientSecret)
}
doc.SetUserAgent("Go-Search(http://go-search.org/)")
}
func TestGddo(t *testing.T) {
doc.SetGithubCredentials("94446b37edb575accd8b",
"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
doc.SetUserAgent("Go-Code-Search-Agent")
pkg := "github.com/daviddengcn/gcse"
p, err := CrawlPackage(http.DefaultClient, pkg, "")
if err != nil {
t.Error(err)
} else {
t.Logf("p: %+v", p.Exported)
}
// t.Error(nil)
}
func init() {
conf, _ := ljconf.Load("conf.json")
serverAddr = conf.String("host", serverAddr)
restSeconds = conf.Int("rest_seconds", restSeconds)
proxyServer = conf.String("proxy", proxyServer)
entriesPerLoop = conf.Int("entries_per_loop", entriesPerLoop)
maxLoops = conf.Int("max_loops", maxLoops)
doBlackPackages = conf.Bool("black_packages.enabled", doBlackPackages)
if doBlackPackages {
blackPkgFn = villa.Path(conf.String("black_packages.filename", blackPkgFn.S()))
}
doc.SetGithubCredentials("94446b37edb575accd8b",
"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
doc.SetUserAgent("Go-Code-Search-Agent")
}
func TestCrawlPackage(t *testing.T) {
if CrawlerGithubClientID != "" {
t.Logf("Github clientid: %s", CrawlerGithubClientID)
t.Logf("Github clientsecret: %s", CrawlerGithubClientSecret)
doc.SetGithubCredentials(CrawlerGithubClientID, CrawlerGithubClientSecret)
}
pkg := "github.com/daviddengcn/gcse"
httpClient := GenHttpClient("")
p, err := CrawlPackage(httpClient, pkg, "")
if err != nil {
if strings.Index(err.Error(), "403") == -1 {
t.Error(err)
}
} else {
assert.Equal(t, "pkg", p.Package, pkg)
}
// pkg = "git.gitorious.org/go-pkg/epubgo.git"
// p, err = CrawlPackage(httpClient, pkg, "")
// if err != nil {
// if strings.Index(err.Error(), "403") == -1 {
// t.Error(err)
// }
// } else {
// assert.Equal(t, "pkg", p.Package, pkg)
// }
pkg = "thezombie.net/libgojira"
p, err = CrawlPackage(httpClient, pkg, "")
if err != nil {
if !IsBadPackage(err) {
t.Errorf("%s should be an invalid package", pkg)
}
} else {
t.Errorf("%s should be an invalid package", pkg)
}
}
func init() {
doc.SetGithubCredentials("94446b37edb575accd8b",
"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
doc.SetUserAgent("Go-Code-Search-Engine")
}