func (s *Spider) analyse() {
go func() {
for p := range s.pageChan {
_analyser := s.analyserPool.Get()
go func(p *page.Page) {
defer s.analyserPool.Release(_analyser)
res := _analyser.Analyse(p)
for _, pipeline := range s.pipelines {
pipeline.Write(res, "")
}
for _, r := range p.NewReqs {
s._scheduler.Add(request.New(r, p.Req.Depth+1))
}
}(p)
}
}()
}
func (s *Spider) AddRequest(req *http.Request) *Spider {
_req := request.New(req, 0)
s.reqs = append(s.reqs, _req)
return s
}