func (dl *myPageDownloader) Download(req base.Request) (*base.Response, error) {
httpReq := req.HttpReq()
log.Info("Do the request (url=%s)... \n", httpReq.URL)
httpResp, err := dl.httpClient.Do(httpReq)
if err != nil {
return nil, err
}
return base.NewResponse(httpResp, req.Depth()), nil
}
func record(level byte, content string) {
if content == "" {
return
}
switch level {
case 0:
log.Info(content)
case 1:
log.Warnf(content)
case 2:
log.Error(content)
}
}
func (analyzer *myAnalyzer) Analyze(
respParsers []ParseResponse,
resp base.Response) (dataList []base.Data, errorList []error) {
if respParsers == nil {
err := errors.New("The response parser list is invalid!")
return nil, []error{err}
}
httpResp := resp.HttpResp()
if httpResp == nil {
err := errors.New("The http response is invalid!")
return nil, []error{err}
}
var reqUrl *url.URL = httpResp.Request.URL
log.Info("Parse the response (reqUrl=%s)... \n", reqUrl)
respDepth := resp.Depth()
// 解析HTTP响应。
dataList = make([]base.Data, 0)
errorList = make([]error, 0)
for i, respParser := range respParsers {
if respParser == nil {
err := errors.New(fmt.Sprintf("The document parser [%d] is invalid!", i))
errorList = append(errorList, err)
continue
}
pDataList, pErrorList := respParser(httpResp, respDepth)
if pDataList != nil {
for _, pData := range pDataList {
dataList = appendDataList(dataList, pData, respDepth)
}
}
if pErrorList != nil {
for _, pError := range pErrorList {
errorList = appendErrorList(errorList, pError)
}
}
}
return dataList, errorList
}