// choose http GET/method to download
func connectByHttp(p *page.Page, req *context.Request) (*http.Response, error) {
client := &http.Client{
CheckRedirect: req.GetRedirectFunc(),
}
httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata()))
if header := req.GetHeader(); header != nil {
httpreq.Header = req.GetHeader()
}
if cookies := req.GetCookies(); cookies != nil {
for i := range cookies {
httpreq.AddCookie(cookies[i])
}
}
var resp *http.Response
if resp, err = client.Do(httpreq); err != nil {
if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" {
// normal
} else {
log.Error(err.Error())
p.SetStatus(true, err.Error())
//fmt.Printf("client do error %v \r\n", err)
return nil, err
}
}
return resp, nil
}
func (self *PageDownloader) Download(req *context.Request) (resp *context.Response, err error) {
var httpResp *http.Response
url := req.GetUrl()
b := req.GetPostBody()
method := req.GetMethod()
switch method {
case "POST", "post", "Post":
httpResp, err = self.client.PostForm(url, b)
if err != nil {
return nil, err
}
case "GET", "get", "Get":
fallthrough
default:
httpResp, err = self.client.Get(url)
if err != nil {
return nil, err
}
}
return context.NewResponse(req, httpResp), nil
}