Example #1
0
func TestMaxWorkers(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, 100, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.quickmeme.com")
	} else {
		fmt.Println(err)
	}
}
Example #2
0
func TestMaxDownloads(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, 1000)
	if err == nil {
		spider.Crawl("http://www.spam.com")
	} else {
		fmt.Println(err)
	}
}
Example #3
0
func TestMaxGoRoutines(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.salami.com")
	} else {
		fmt.Println(err)
	}
}