// Construct a worker with given settings.
func NewWorker(settings *ss.ScanSettings,
factory client.ClientFactory,
src <-chan *url.URL,
adder workqueue.QueueAddFunc,
done workqueue.QueueDoneFunc,
rchan chan<- results.Result) *Worker {
w := &Worker{
client: factory.Get(),
settings: settings,
src: src,
adder: adder,
done: done,
rchan: rchan,
stop: make(chan bool),
}
// Install redirect handler
redirHandler := func(req *http.Request, _ []*http.Request) error {
w.redir = req
return fmt.Errorf("Stop redirect.")
}
w.client.SetCheckRedirect(redirHandler)
return w
}
func GetRobotsForURL(target *url.URL, factory client.ClientFactory) (*RobotsData, error) {
client := factory.Get()
ref, _ := url.Parse("/robots.txt")
robotsURL := target.ResolveReference(ref)
resp, err := client.RequestURL(robotsURL)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return ParseRobotsTxt(body)
}