//!+crawl
func crawl(url string) []string {
fmt.Println(url)
list, err := links.Extract(url)
if err != nil {
log.Print(err)
}
return list
}
func crawl(url string) []string {
fmt.Println(url)
tokens <- struct{}{} // acquire a token
list, err := links.Extract(url)
<-tokens // release the token
if err != nil {
log.Print(err)
}
return list
}
func Crawl(url string) []string {
fmt.Println(url)
// TODO: BobK I still don't get the struct{}{} syntax
tokens <- struct{}{} // acquire a token
list, err := links.Extract(url)
<-tokens // release the token
if err != nil {
log.Print(err)
}
return list
}
func crawl(rawurl string) func(item string) []string {
fmt.Println(rawurl)
u, err := url.Parse(rawurl)
if err != nil {
log.Print(err)
}
tempdir := os.TempDir()
return func(item string) []string {
fmt.Println(item)
itemUrl, err := url.Parse(item)
if err != nil {
log.Print(err)
}
// if same host, save contents on disk.
if itemUrl.Host == u.Host {
file, err := ioutil.TempFile(tempdir, "crawler-")
defer file.Close()
if err != nil {
log.Print(err)
}
fmt.Printf("write %s to %s\n", item, file.Name())
resp, err := http.Get(item)
defer resp.Body.Close()
if err != nil {
log.Print(err)
}
if _, err := io.Copy(file, resp.Body); err != nil {
log.Print(err)
}
}
list, err := links.Extract(item)
if err != nil {
log.Print(err)
}
return list
}
}