Golang Spider.Crawl Examples

Example #1

File: blackbox_test.go Project: nightowlware/cybergrab

func TestMaxWorkers(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, 100, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.quickmeme.com")
	} else {
		fmt.Println(err)
	}
}

Example #2

File: blackbox_test.go Project: nightowlware/cybergrab

func TestMaxDownloads(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, 1000)
	if err == nil {
		spider.Crawl("http://www.spam.com")
	} else {
		fmt.Println(err)
	}
}

Example #3

File: blackbox_test.go Project: nightowlware/cybergrab

func TestMaxGoRoutines(t *testing.T) {
	var spider cybergrab.Spider

	spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, cybergrab.MAX_WORKERS)
	if err == nil {
		spider.Crawl("http://www.salami.com")
	} else {
		fmt.Println(err)
	}
}