func (this *HttpDownloader) downloadJson(p *page.Page, req *request.Request) *page.Page { var err error p, destbody := this.downloadFile(p, req) if !p.IsSucc() { return p } var body []byte body = []byte(destbody) mtype := req.GetResponceType() if mtype == "jsonp" { tmpstr := util.JsonpToJson(destbody) body = []byte(tmpstr) } var r *simplejson.Json if r, err = simplejson.NewJson(body); err != nil { mlog.LogInst().LogError(string(body) + "\t" + err.Error()) p.SetStatus(true, err.Error()) return p } // json result p.SetBodyStr(string(body)).SetJson(r).SetStatus(false, "") return p }
func (this *HttpDownloader) downloadJson(req *request.Request) *page.Page { var err error var url string if url = req.GetUrl(); len(url) == 0 { mlog.LogInst().LogError(err.Error()) return nil } var resp *http.Response if resp, err = http.Get(url); err != nil { mlog.LogInst().LogError(err.Error()) return nil } defer resp.Body.Close() var body []byte if body, err = ioutil.ReadAll(resp.Body); err != nil { mlog.LogInst().LogError(err.Error()) return nil } var r *simplejson.Json if r, err = simplejson.NewJson(body); err != nil { mlog.LogInst().LogError(err.Error()) return nil } // create Page // json result var p *page.Page = page.NewPage(req). SetBodyStr(string(body)). SetJson(r) return p }
func (this *HttpDownloader) downloadHtml(req *request.Request) *page.Page { var err error var url string if url = req.GetUrl(); len(url) == 0 { return nil } var resp *http.Response if resp, err = http.Get(url); err != nil { mlog.LogInst().LogError(err.Error()) return nil } defer resp.Body.Close() var doc *goquery.Document if doc, err = goquery.NewDocumentFromReader(resp.Body); err != nil { mlog.LogInst().LogError(err.Error()) return nil } var body string if body, err = doc.Html(); err != nil { mlog.LogInst().LogError(err.Error()) return nil } // create Page var p *page.Page = page.NewPage(req). SetBodyStr(body). SetHtmlParser(doc) return p }
// add Request to Schedule func (this *Spider) addRequest(req *request.Request) { if req == nil { mlog.LogInst().LogError("request is nil") return } else if req.GetUrl() == "" { mlog.LogInst().LogError("request is empty") return } this.pScheduler.Push(req) }
func (this *HttpDownloader) Download(req *request.Request) *page.Page { var mtype string mtype = req.GetResponceType() switch mtype { case "html": return this.downloadHtml(req) case "json": return this.downloadJson(req) default: mlog.LogInst().LogError("error request type:" + mtype) return nil } }
func (this *RedisScheduler) Push(requ *request.Request) { this.locker.Lock() defer this.locker.Unlock() requJson, err := json.Marshal(requ) if err != nil { mlog.LogInst().LogError("RedisScheduler Push Error: " + err.Error()) return } conn := this.redisPool.Get() defer conn.Close() if err != nil { mlog.LogInst().LogError("RedisScheduler Push Error: " + err.Error()) return } if this.forbiddenDuplicateUrl { urlExist, err := conn.Do("HGET", this.urlList, requ.GetUrl()) if err != nil { mlog.LogInst().LogError("RedisScheduler Push Error: " + err.Error()) return } if urlExist != nil { return } conn.Do("MULTI") _, err = conn.Do("HSET", this.urlList, requ.GetUrl(), 1) if err != nil { mlog.LogInst().LogError("RedisScheduler Push Error: " + err.Error()) conn.Do("DISCARD") return } } _, err = conn.Do("RPUSH", this.requestList, requJson) if err != nil { mlog.LogInst().LogError("RedisScheduler Push Error: " + err.Error()) if this.forbiddenDuplicateUrl { conn.Do("DISCARD") } return } if this.forbiddenDuplicateUrl { conn.Do("EXEC") } }
func (this *QueueScheduler) Push(requ *request.Request) { this.locker.Lock() var key [md5.Size]byte if this.rm { key = md5.Sum([]byte(requ.GetUrl())) if _, ok := this.rmKey[key]; ok { this.locker.Unlock() return } } e := this.queue.PushBack(requ) if this.rm { this.rmKey[key] = e } this.locker.Unlock() }
func (this *HttpDownloader) Download(req *request.Request) *page.Page { var mtype string var p = page.NewPage(req) mtype = req.GetResponceType() switch mtype { case "html": return this.downloadHtml(p, req) case "json": fallthrough case "jsonp": return this.downloadJson(p, req) case "text": return this.downloadText(p, req) default: mlog.LogInst().LogError("error request type:" + mtype) } return p }
// choose a proxy server to excute http GET/method to download func connectByHttpProxy(p *page.Page, in_req *request.Request) (*http.Response, error) { request, _ := http.NewRequest("GET", in_req.GetUrl(), nil) proxy, err := url.Parse(in_req.GetProxyHost()) if err != nil { return nil, err } client := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyURL(proxy), }, } resp, err := client.Do(request) if err != nil { return nil, err } return resp, nil }
// Download file and change the charset of page charset. func (this *HttpDownloader) downloadFile(p *page.Page, req *request.Request) (*page.Page, string) { var err error var urlstr string if urlstr = req.GetUrl(); len(urlstr) == 0 { mlog.LogInst().LogError("url is empty") p.SetStatus(true, "url is empty") return p, "" } var resp *http.Response if proxystr := req.GetProxyHost(); len(proxystr) != 0 { //using http proxy //fmt.Print("HttpProxy Enter ",proxystr,"\n") resp, err = connectByHttpProxy(p, req) } else { //normal http download //fmt.Print("Http Normal Enter \n",proxystr,"\n") resp, err = connectByHttp(p, req) } if err != nil { return p, "" } //b, _ := ioutil.ReadAll(resp.Body) //fmt.Printf("Resp body %v \r\n", string(b)) p.SetHeader(resp.Header) p.SetCookies(resp.Cookies()) // get converter to utf-8 var bodyStr string if resp.Header.Get("Content-Encoding") == "gzip" { bodyStr = this.changeCharsetEncodingAutoGzipSupport(resp.Header.Get("Content-Type"), resp.Body) } else { bodyStr = this.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body) } //fmt.Printf("utf-8 body %v \r\n", bodyStr) defer resp.Body.Close() return p, bodyStr }
// choose http GET/method to download func connectByHttp(p *page.Page, req *request.Request) (*http.Response, error) { client := &http.Client{ CheckRedirect: req.GetRedirectFunc(), } httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata())) if header := req.GetHeader(); header != nil { httpreq.Header = req.GetHeader() } if cookies := req.GetCookies(); cookies != nil { for i := range cookies { httpreq.AddCookie(cookies[i]) } } var resp *http.Response if resp, err = client.Do(httpreq); err != nil { if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" { // normal } else { mlog.LogInst().LogError(err.Error()) p.SetStatus(true, err.Error()) //fmt.Printf("client do error %v \r\n", err) return nil, err } } return resp, nil }
// Download file and change the charset of page charset. func (this *HttpDownloader) downloadFile(p *page.Page, req *request.Request) (*page.Page, string) { var err error var urlstr string if urlstr = req.GetUrl(); len(urlstr) == 0 { mlog.LogInst().LogError("url is empty") p.SetStatus(true, "url is empty") return p, "" } client := &http.Client{ CheckRedirect: req.GetRedirectFunc(), } httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata())) if header := req.GetHeader(); header != nil { httpreq.Header = req.GetHeader() } if cookies := req.GetCookies(); cookies != nil { for i := range cookies { httpreq.AddCookie(cookies[i]) } } var resp *http.Response if resp, err = client.Do(httpreq); err != nil { if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" { // normal } else { mlog.LogInst().LogError(err.Error()) p.SetStatus(true, err.Error()) return p, "" } } p.SetHeader(resp.Header) p.SetCookies(resp.Cookies()) // get converter to utf-8 bodyStr := this.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body) defer resp.Body.Close() return p, bodyStr }