Beispiel #1
0
// choose http GET/method to download
func connectByHttp(p *page.Page, req *context.Request) (*http.Response, error) {
	client := &http.Client{
		CheckRedirect: req.GetRedirectFunc(),
	}

	httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata()))
	if header := req.GetHeader(); header != nil {
		httpreq.Header = req.GetHeader()
	}

	if cookies := req.GetCookies(); cookies != nil {
		for i := range cookies {
			httpreq.AddCookie(cookies[i])
		}
	}

	var resp *http.Response
	if resp, err = client.Do(httpreq); err != nil {
		if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" {
			//  normal
		} else {
			log.Error(err.Error())
			p.SetStatus(true, err.Error())
			//fmt.Printf("client do error %v \r\n", err)
			return nil, err
		}
	}

	return resp, nil
}
Beispiel #2
0
func (self *PageDownloader) Download(req *context.Request) (resp *context.Response, err error) {
	var httpResp *http.Response

	url := req.GetUrl()
	b := req.GetPostBody()

	method := req.GetMethod()
	switch method {
	case "POST", "post", "Post":
		httpResp, err = self.client.PostForm(url, b)
		if err != nil {
			return nil, err
		}
	case "GET", "get", "Get":
		fallthrough
	default:
		httpResp, err = self.client.Get(url)
		if err != nil {
			return nil, err
		}
	}

	return context.NewResponse(req, httpResp), nil
}