Beispiel #1
0
func crawl(url string) []string {
	fmt.Println(url)
	list, err := links.Extract(url)
	if err != nil {
		log.Print(err)
	}
	return list
}
Beispiel #2
0
func crawl(url string) []string {
	fmt.Println(url)
	tokens <- struct{}{} // acquire a token
	list, err := links.Extract(url)
	<-tokens // release the token
	if err != nil {
		log.Print(err)
	}
	return list
}
Beispiel #3
0
func crawl(theURL string) []string {
	fmt.Println(theURL)
	list, err := links.Extract(theURL)
	if err != nil {
		log.Print(err)
	}
	list = selectSameDomain(theURL, list)

	for _, link := range list {
		save(link)
	}

	return list
}
Beispiel #4
0
func crawl(url string) []string {
	fmt.Println(url)
	tokens <- struct{}{} // acquire a token
	list, err := links.Extract(url)
	<-tokens // release the token
	if err != nil {
		log.Print(err)
	}

	list = selectSameDomain(url, list)
	for _, link := range list {
		save(link)
	}

	return list
}