// 序列化
func (self *Request) Serialize() string {
for k, v := range self.Temp {
self.Temp.Set(k, v)
self.TempIsJson[k] = true
}
b, _ := json.Marshal(self)
return strings.Replace(util.Bytes2String(b), `\u0026`, `&`, -1)
}
func (self Temp) Set(key string, value interface{}) Temp {
b, err := json.Marshal(value)
if err != nil {
logs.Log.Error(" * Request.Temp.Set(%v): %v", key, err)
}
self[key] = util.Bytes2String(b)
return self
}
// GetBodyStr returns plain string crawled.
func (self *Context) initText() {
defer self.Response.Body.Close()
// get converter to utf-8
// Charset auto determine. Use golang.org/x/net/html/charset. Get response body and change it to utf-8
destReader, err := charset.NewReader(self.Response.Body, self.Response.Header.Get("Content-Type"))
if err != nil {
logs.Log.Warning(err.Error())
destReader = self.Response.Body
}
sorbody, err := ioutil.ReadAll(destReader)
if err != nil {
logs.Log.Error(err.Error())
return
// For gb2312, an error will be returned.
// Error like: simplifiedchinese: invalid GBK encoding
}
//e,name,certain := charset.DetermineEncoding(sorbody,self.Response.Header.Get("Content-Type"))
self.text = util.Bytes2String(sorbody)
}
// GetBodyStr returns plain string crawled.
func (self *Context) GetText() string {
if self.text == nil {
self.initText()
}
return util.Bytes2String(self.text)
}