// Deal with several urls and return the PageItems slice.
func (this *Spider) GetAll(urls []string, respType string) []*page_items.PageItems {
for _, u := range urls {
req := request.NewRequest(u, respType, "", "GET", "", nil, nil, nil, nil)
this.AddRequest(req)
}
pip := pipeline.NewCollectPipelinePageItems()
this.AddPipeline(pip)
this.Run()
return pip.GetCollected()
}
// Deal with several urls and return the PageItems slice
func (this *Spider) GetAllByRequest(reqs []*request.Request) []*page_items.PageItems {
// push url
for _, req := range reqs {
//req := request.NewRequest(u, respType, urltag, method, postdata, header, cookies)
this.AddRequest(req)
}
pip := pipeline.NewCollectPipelinePageItems()
this.AddPipeline(pip)
this.Run()
return pip.GetCollected()
}