//下载文件,并对字符编码做相应的处理 func (this *HttpDownLoader) downloadFile(p *page.Page, req *page.Request) (*page.Page, string) { var err error var httpResp *http.Response var urlStr string var method string urlStr = req.GetUrl() if len(urlStr) == 0 { logs.GetFirstLogger().Error("url is empty") p.SetStatus(true, "url is empty") return p, "" } method = req.GetMethod() if method == "POST" { httpResp, err = http.Post(req.GetUrl(), "application/x-www-form-urlencoded", strings.NewReader(req.GetPostData())) } else { httpResp, err = http.Get(req.GetUrl()) } if err != nil { logs.GetFirstLogger().Error("http visit error :" + err.Error()) p.SetStatus(true, err.Error()) } p.SetHeader(httpResp.Header) p.SetCookies(httpResp.Cookies()) body, _ := ioutil.ReadAll(httpResp.Body) bodyStr := string(body) defer httpResp.Body.Close() return p, bodyStr }