示例#1
0
文件: crawl.go 项目: pombredanne/gcse
func init() {
	gob.Register(CrawlingEntry{})

	doc.SetGithubCredentials("94446b37edb575accd8b",
		"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
	doc.SetUserAgent("Go-Code-Search-Agent")
}
示例#2
0
文件: cmain.go 项目: MarkBruns/gcse
func init() {
	if gcse.CrawlerGithubClientID != "" {
		log.Printf("Github clientid: %s", gcse.CrawlerGithubClientID)
		log.Printf("Github clientsecret: %s", gcse.CrawlerGithubClientSecret)
		doc.SetGithubCredentials(gcse.CrawlerGithubClientID, gcse.CrawlerGithubClientSecret)
	}
	doc.SetUserAgent("Go-Search(http://go-search.org/)")
}
示例#3
0
func TestGddo(t *testing.T) {
	doc.SetGithubCredentials("94446b37edb575accd8b",
		"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
	doc.SetUserAgent("Go-Code-Search-Agent")

	pkg := "github.com/daviddengcn/gcse"
	p, err := CrawlPackage(http.DefaultClient, pkg, "")
	if err != nil {
		t.Error(err)
	} else {
		t.Logf("p: %+v", p.Exported)
	}
	//	t.Error(nil)
}
示例#4
0
func init() {
	conf, _ := ljconf.Load("conf.json")

	serverAddr = conf.String("host", serverAddr)
	restSeconds = conf.Int("rest_seconds", restSeconds)
	proxyServer = conf.String("proxy", proxyServer)

	entriesPerLoop = conf.Int("entries_per_loop", entriesPerLoop)
	maxLoops = conf.Int("max_loops", maxLoops)

	doBlackPackages = conf.Bool("black_packages.enabled", doBlackPackages)
	if doBlackPackages {
		blackPkgFn = villa.Path(conf.String("black_packages.filename", blackPkgFn.S()))
	}

	doc.SetGithubCredentials("94446b37edb575accd8b",
		"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
	doc.SetUserAgent("Go-Code-Search-Agent")
}
示例#5
0
func TestCrawlPackage(t *testing.T) {
	if CrawlerGithubClientID != "" {
		t.Logf("Github clientid: %s", CrawlerGithubClientID)
		t.Logf("Github clientsecret: %s", CrawlerGithubClientSecret)
		doc.SetGithubCredentials(CrawlerGithubClientID, CrawlerGithubClientSecret)
	}

	pkg := "github.com/daviddengcn/gcse"
	httpClient := GenHttpClient("")
	p, err := CrawlPackage(httpClient, pkg, "")
	if err != nil {
		if strings.Index(err.Error(), "403") == -1 {
			t.Error(err)
		}
	} else {
		assert.Equal(t, "pkg", p.Package, pkg)
	}

	//	pkg = "git.gitorious.org/go-pkg/epubgo.git"
	//	p, err = CrawlPackage(httpClient, pkg, "")
	//	if err != nil {
	//		if strings.Index(err.Error(), "403") == -1 {
	//			t.Error(err)
	//		}
	//	} else {
	//		assert.Equal(t, "pkg", p.Package, pkg)
	//	}

	pkg = "thezombie.net/libgojira"
	p, err = CrawlPackage(httpClient, pkg, "")
	if err != nil {
		if !IsBadPackage(err) {
			t.Errorf("%s should be an invalid package", pkg)
		}
	} else {
		t.Errorf("%s should be an invalid package", pkg)
	}
}
示例#6
0
func init() {
	doc.SetGithubCredentials("94446b37edb575accd8b",
		"15f55815f0515a3f6ad057aaffa9ea83dceb220b")
	doc.SetUserAgent("Go-Code-Search-Engine")
}