func (this *QuickEngineProcesser) processRequests(resp *common.Response, y *common.Yield, rule _Rule) { var TrimFunc extractor.TrimFunc switch rule.RequestRule.TrimFunc { case "trim_html_tags": TrimFunc = extractor.TrimHtmlTags case "trim_blank": TrimFunc = extractor.TrimBlank } items := extractor.NewExtractor(). SetScopeRule(rule.RequestRule.ScopeRule). SetRules(rule.RequestRule.KVRule). SetTrimFunc(TrimFunc). Extract(resp) for _, item := range items { for _, url := range item.GetAll() { if strings.HasPrefix(url, "http://") { y.AddRequest(common.NewRequest(url)) } else { y.AddRequest(common.NewRequest(rule.BaseUrl + url)) } } } }
func (this *MyProcesser) processNext(resp *common.Response, y *common.Yield) { m := regexp.MustCompile(`(?s)<span class="next">.*?<a href="(.*?)"`).FindStringSubmatch(resp.Body) if len(m) > 0 { y.AddRequest(common.NewRequest(this.baseUrl + m[1])) } }