Пример #1
0
// Parse html dom here and record the parse result that we want to crawl.
// Package simplejson (https://github.com/bitly/go-simplejson) is used to parse data of json.
func (this *MyPageProcesser) Process(p *page.Page) {
	if !p.IsSucc() {
		println(p.Errormsg())
		return
	}

	query := p.GetJson()
	status, err := query.GetPath("result", "status", "code").Int()
	if status != 0 || err != nil {
		log.Panicf("page is crawled error : errorinfo=%s : status=%d : startNewsId=%d", err.Error(), status, this.startNewsId)
	}
	num, err := query.GetPath("result", "pageStr", "pageSize").Int()
	if num == 0 || err != nil {
		// Add url of next crawl
		startIdstr := strconv.Itoa(this.startNewsId)
		p.AddTargetRequest("http://live.sina.com.cn/zt/api/l/get/finance/globalnews1/index.htm?format=json&id="+startIdstr+"&pagesize=10&dire=f", "json")
		return
	}

	var idint, nextid int
	var nextidstr string
	query = query.Get("result").Get("data")
	for i := 0; i < num; i++ {
		id, err := query.GetIndex(i).Get("id").String()
		if id == "" || err != nil {
			continue
		}
		idint, err = strconv.Atoi(id)
		if err != nil {
			continue
		}
		if idint <= this.startNewsId {
			break
		}
		if i == 0 {
			nextid = idint
			nextidstr = id
		}
		content, err := query.GetIndex(i).Get("content").String()
		if content == "" || err != nil {
			continue
		}
		time, err := query.GetIndex(i).Get("created_at").String()
		if err != nil {
			continue
		}

		p.AddField(id+"_id", id)
		p.AddField(id+"_content", content)
		p.AddField(id+"_time", time)
	}
	// Add url of next crawl
	this.startNewsId = nextid
	p.AddTargetRequest("http://live.sina.com.cn/zt/api/l/get/finance/globalnews1/index.htm?format=json&id="+nextidstr+"&pagesize=10&dire=f", "json")
	//println(p.GetTargetRequests())

}
Пример #2
0
// Parse html dom here and record the parse result that we want to crawl.
// Package goquery (http://godoc.org/github.com/PuerkitoBio/goquery) is used to parse html.
func (this *MyPageProcesser) Process(p *page.Page) {
	if !p.IsSucc() {
		println(p.Errormsg())
		return
	}

	html := p.GetBodyStr()
	newUrls := urlutil.GetAllUrlIn(p.GetRequest().GetUrl(), html)
	for _, newUrl := range newUrls {
		newUrl = strings.Replace(newUrl, "//weibo.com/", "//tw.weibo.com/", -1)
		p.AddTargetRequest(newUrl, "html")
	}

	mailAddrList := mailaddrutil.GetAllMailAddrIn(html)
	for _, mailAddr := range mailAddrList {
		if _, ok := this.mailAddrMap[mailAddr]; !ok {
			this.mailAddrMap[mailAddr] = true
			this.mailLogger.WriteString(mailAddr + "\n")
			this.MailHandle.Push(mailAddr)
		}
	}
}