Example #1
0
func (s *Spider) analyse() {
	go func() {
		for p := range s.pageChan {
			_analyser := s.analyserPool.Get()
			go func(p *page.Page) {
				defer s.analyserPool.Release(_analyser)
				res := _analyser.Analyse(p)
				for _, pipeline := range s.pipelines {
					pipeline.Write(res, "")
				}

				for _, r := range p.NewReqs {
					s._scheduler.Add(request.New(r, p.Req.Depth+1))
				}
			}(p)
		}

	}()
}
Example #2
0
func (s *Spider) AddRequest(req *http.Request) *Spider {
	_req := request.New(req, 0)
	s.reqs = append(s.reqs, _req)

	return s
}