// choose a proxy server to excute http GET/method to download func connectByHttpProxy(p *robot.Page, in_req *robot.Request) (*http.Response, error) { request, _ := http.NewRequest("GET", in_req.GetUrl(), nil) proxy, err := url.Parse(in_req.GetProxyHost()) if err != nil { return nil, err } client := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyURL(proxy), }, } resp, err := client.Do(request) if err != nil { return nil, err } return resp, nil }
// Download file and change the charset of page charset. func (self *HttpDownloader) downloadFile(p *robot.Page, req *robot.Request) (*robot.Page, string) { var err error var urlstr string if urlstr = req.GetUrl(); len(urlstr) == 0 { mlog.LogInst().LogError("url is empty") p.SetStatus(true, "url is empty") return p, "" } var resp *http.Response if proxystr := req.GetProxyHost(); len(proxystr) != 0 { //using http proxy //fmt.Print("HttpProxy Enter ",proxystr,"\n") resp, err = connectByHttpProxy(p, req) } else { //normal http download //fmt.Print("Http Normal Enter \n",proxystr,"\n") resp, err = connectByHttp(p, req) } if err != nil { return p, "" } p.SetHeader(resp.Header) p.SetCookies(resp.Cookies()) // get converter to utf-8 var bodyStr string if resp.Header.Get("Content-Encoding") == "gzip" { bodyStr = self.changeCharsetEncodingAutoGzipSupport(resp.Header.Get("Content-Type"), resp.Body) } else { bodyStr = self.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body) } //fmt.Printf("utf-8 body %v \r\n", bodyStr) defer resp.Body.Close() return p, bodyStr }