func (this *HttpDownloader) downloadJson(req *request.Request) *page.Page {
var err error
var url string
if url = req.GetUrl(); len(url) == 0 {
mlog.LogInst().LogError(err.Error())
return nil
}
var resp *http.Response
if resp, err = http.Get(url); err != nil {
mlog.LogInst().LogError(err.Error())
return nil
}
defer resp.Body.Close()
var body []byte
if body, err = ioutil.ReadAll(resp.Body); err != nil {
mlog.LogInst().LogError(err.Error())
return nil
}
var r *simplejson.Json
if r, err = simplejson.NewJson(body); err != nil {
mlog.LogInst().LogError(err.Error())
return nil
}
// create Page
// json result
var p *page.Page = page.NewPage(req).
SetBodyStr(string(body)).
SetJson(r)
return p
}
func (this *HttpDownloader) downloadHtml(req *request.Request) *page.Page {
var err error
var url string
if url = req.GetUrl(); len(url) == 0 {
return nil
}
var resp *http.Response
if resp, err = http.Get(url); err != nil {
mlog.LogInst().LogError(err.Error())
return nil
}
defer resp.Body.Close()
var doc *goquery.Document
if doc, err = goquery.NewDocumentFromReader(resp.Body); err != nil {
mlog.LogInst().LogError(err.Error())
return nil
}
var body string
if body, err = doc.Html(); err != nil {
mlog.LogInst().LogError(err.Error())
return nil
}
// create Page
var p *page.Page = page.NewPage(req).
SetBodyStr(body).
SetHtmlParser(doc)
return p
}
func (this *HttpDownloader) Download(req *request.Request) *page.Page {
var mtype string
var p = page.NewPage(req)
mtype = req.GetResponceType()
switch mtype {
case "html":
return this.downloadHtml(p, req)
case "json":
fallthrough
case "jsonp":
return this.downloadJson(p, req)
case "text":
return this.downloadText(p, req)
default:
mlog.LogInst().LogError("error request type:" + mtype)
}
return p
}