func (this *ExampleExtender) Visit(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
// fmt.Println("visit url: ", ctx.URL(), "state: ", ctx.State)
fmt.Printf("\"%v\",\n", ctx.URL())
fmt.Printf("\t%T\t%+v\n", doc, doc)
// urls := processLinks(doc)
links := make(map[*url.URL]interface{})
// i, _ := ctx.State.(int)
// nextDepth := i - 1
// if nextDepth <= 0 {
// return nil, false
// }
// for _, u := range urls {
// links[u] = nextDepth
// }
return links, false
}
func (e *Ext) Filter(ctx *gocrawl.URLContext, isVisited bool) bool {
if isVisited {
return false
}
if ctx.URL().Host == "github.com" || ctx.URL().Host == "golang.org" || ctx.URL().Host == "0value.com" {
return true
}
return false
}
func (e *Ext) Visit(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
fmt.Printf("Visit: %s\n", ctx.URL())
return nil, true
}