// Process the response for a URL.
func (this *worker) visitUrl(res *http.Response) []*url.URL {
var doc *goquery.Document
var harvested []*url.URL
var doLinks bool
// Load a goquery document and call the visitor function
if node, e := html.Parse(res.Body); e != nil {
this.logFunc(LogError, "ERROR parsing %s: %s\n", res.Request.URL.String(), e.Error())
} else {
doc = goquery.NewDocumentFromNode(node)
doc.Url = res.Request.URL
}
// Visit the document (with nil goquery doc if failed to load)
if this.visitor != nil {
if harvested, doLinks = this.visitor(res, doc); doLinks && doc != nil {
// Links were not processed by the visitor, so process links
harvested = this.processLinks(doc)
}
} else {
this.logFunc(LogInfo, "missing visitor function: %s\n", res.Request.URL.String())
}
return harvested
}
// Process the response for a URL.
func (this *worker) visitUrl(res *http.Response) []*url.URL {
var doc *goquery.Document
var harvested []*url.URL
var doLinks bool
// Load a goquery document and call the visitor function
if bd, e := ioutil.ReadAll(res.Body); e != nil {
this.extender.Error(newCrawlError(e, CekReadBody, res.Request.URL))
this.logFunc(LogError, "ERROR reading body %s: %s", res.Request.URL.String(), e.Error())
} else {
if node, e := html.Parse(bytes.NewBuffer(bd)); e != nil {
this.extender.Error(newCrawlError(e, CekParseBody, res.Request.URL))
this.logFunc(LogError, "ERROR parsing %s: %s", res.Request.URL.String(), e.Error())
} else {
doc = goquery.NewDocumentFromNode(node)
doc.Url = res.Request.URL
}
// Re-assign the body so it can be consumed by the visitor function
res.Body = ioutil.NopCloser(bytes.NewBuffer(bd))
}
// Visit the document (with nil goquery doc if failed to load)
if harvested, doLinks = this.extender.Visit(res, doc); doLinks {
// Links were not processed by the visitor, so process links
if doc != nil {
harvested = this.processLinks(doc)
} else {
this.extender.Error(newCrawlErrorMessage("No goquery document to process links.", CekProcessLinks, res.Request.URL))
this.logFunc(LogError, "ERROR processing links %s", res.Request.URL.String())
}
}
// Notify that this URL has been visited
this.extender.Visited(res.Request.URL, harvested)
return harvested
}