func (dl *myPageDownloader) Download(req base.Request) (*base.Response, error) {
httpReq := req.HttpReq()
logger.Infof("Do the request (url=%s)... \n", httpReq.URL)
httpResp, err := dl.httpClient.Do(httpReq)
if err != nil {
return nil, err
}
return base.NewResponse(httpResp, req.Depth()), nil
}
// 把请求存放到请求缓存。
func (sched *myScheduler) saveReqToCache(req base.Request, code string) bool {
httpReq := req.HttpReq()
if httpReq == nil {
logger.Warnln("Ignore the request! It's HTTP request is invalid!")
return false
}
reqUrl := httpReq.URL
if reqUrl == nil {
logger.Warnln("Ignore the request! It's url is is invalid!")
return false
}
if strings.ToLower(reqUrl.Scheme) != "http" {
logger.Warnf("Ignore the request! It's url scheme '%s', but should be 'http'!\n", reqUrl.Scheme)
return false
}
if _, ok := sched.urlMap[reqUrl.String()]; ok {
logger.Warnf("Ignore the request! It's url is repeated. (requestUrl=%s)\n", reqUrl)
return false
}
if pd, _ := getPrimaryDomain(httpReq.Host); pd != sched.primaryDomain {
logger.Warnf("Ignore the request! It's host '%s' not in primary domain '%s'. (requestUrl=%s)\n",
httpReq.Host, sched.primaryDomain, reqUrl)
return false
}
if req.Depth() > sched.crawlDepth {
logger.Warnf("Ignore the request! It's depth %d greater than %d. (requestUrl=%s)\n",
req.Depth(), sched.crawlDepth, reqUrl)
return false
}
if sched.stopSign.Signed() {
sched.stopSign.Deal(code)
return false
}
sched.reqCache.put(&req)
sched.urlMap[reqUrl.String()] = true
return true
}