func crawl() {
var producer *nsq.Producer
var consumer *nsq.Consumer
handler := nsq.HandlerFunc(func(m *nsq.Message) error {
scan := gryffin.NewScanFromJson(m.Body, t)
if delay := scan.RateLimit(); delay != 0 {
go func() {
time.Sleep(time.Duration(delay) * time.Second)
err := producer.Publish("seed", scan.Json())
if err != nil {
fmt.Println("Could not publish", "fuzz", err)
}
}()
} else {
// TODO - phantom JS timeout should be an input argument.
r := &renderer.PhantomJSRenderer{Timeout: 60}
wq <- true
scan.CrawlAsync(r)
go func() {
if s := <-r.GetRequestBody(); s != nil {
// fmt.Println("Got request body", s.Request.URL)
err := producer.Publish("fuzz", s.Json())
if err != nil {
fmt.Println("Could not publish", "fuzz", err)
}
}
}()
go func() {
//
// Renderer will close all channels when a page is duplicated.
// Therefore we don't need to test whether the link is coming
// from a duplicated page or not
for s := range r.GetLinks() {
if ok := s.ApplyLinkRules(); ok {
err := producer.Publish("seed", s.Json())
if err != nil {
fmt.Println("Could not publish", "seed", err)
}
}
}
<-wq
}()
}
return nil
})
producer = newProducer()
defer producer.Stop()
consumer = newConsumer("seed", "primary", handler)
defer consumer.Stop()
wg.Wait()
}
func crawl() {
var producer *nsq.Producer
var consumer *nsq.Consumer
handler := nsq.HandlerFunc(func(m *nsq.Message) error {
scan := gryffin.NewScanFromJson(m.Body, t)
if delay := scan.RateLimit(); delay != 0 {
go func() {
time.Sleep(time.Duration(delay) * time.Second)
err := producer.Publish("seed", scan.Json())
if err != nil {
fmt.Println("Could not publish", "fuzz", err)
}
}()
} else {
// TODO - phantom JS timeout should be an input argument.
r := &renderer.PhantomJSRenderer{Timeout: 60}
wq <- true
scan.CrawlAsync(r)
go func() {
if s := <-r.GetRequestBody(); s != nil {
// fmt.Println("Got request body", s.Request.URL)
err := producer.Publish("fuzz", s.Json())
if err != nil {
fmt.Println("Could not publish", "fuzz", err)
}
}
}()
go func() {
isUnique := false
for s := range r.GetLinks() {
// do the evaluation once only.
isUnique = isUnique || scan.IsUnique()
if isUnique {
if ok := s.ApplyLinkRules(); ok {
err := producer.Publish("seed", s.Json())
if err != nil {
fmt.Println("Could not publish", "seed", err)
}
}
}
}
<-wq
}()
}
return nil
})
producer = newProducer()
defer producer.Stop()
consumer = newConsumer("seed", "primary", handler)
defer consumer.Stop()
wg.Wait()
}
func fuzzWithArachni() {
var consumer *nsq.Consumer
handler := nsq.HandlerFunc(func(m *nsq.Message) error {
wq <- true
scan := gryffin.NewScanFromJson(m.Body, t)
f := &arachni.Fuzzer{}
f.Fuzz(scan)
<-wq
return nil
})
consumer = newConsumer("fuzz", "arachni", handler)
defer consumer.Stop()
wg.Wait()
}