示例#1
0
func crawl() {

	var producer *nsq.Producer
	var consumer *nsq.Consumer

	handler := nsq.HandlerFunc(func(m *nsq.Message) error {
		scan := gryffin.NewScanFromJson(m.Body, t)

		if delay := scan.RateLimit(); delay != 0 {
			go func() {
				time.Sleep(time.Duration(delay) * time.Second)
				err := producer.Publish("seed", scan.Json())
				if err != nil {
					fmt.Println("Could not publish", "fuzz", err)
				}
			}()
		} else {
			// TODO - phantom JS timeout should be an input argument.
			r := &renderer.PhantomJSRenderer{Timeout: 60}
			wq <- true
			scan.CrawlAsync(r)
			go func() {
				if s := <-r.GetRequestBody(); s != nil {
					// fmt.Println("Got request body", s.Request.URL)
					err := producer.Publish("fuzz", s.Json())
					if err != nil {
						fmt.Println("Could not publish", "fuzz", err)
					}
				}
			}()

			go func() {

				//
				// Renderer will close all channels when a page is duplicated.
				// Therefore we don't need to test whether the link is coming
				// from a duplicated page or not
				for s := range r.GetLinks() {
					if ok := s.ApplyLinkRules(); ok {
						err := producer.Publish("seed", s.Json())
						if err != nil {
							fmt.Println("Could not publish", "seed", err)
						}
					}
				}
				<-wq
			}()
		}

		return nil
	})

	producer = newProducer()
	defer producer.Stop()
	consumer = newConsumer("seed", "primary", handler)
	defer consumer.Stop()

	wg.Wait()

}
示例#2
0
文件: main.go 项目: 0xwindows/gryffin
func crawl() {

	var producer *nsq.Producer
	var consumer *nsq.Consumer

	handler := nsq.HandlerFunc(func(m *nsq.Message) error {
		scan := gryffin.NewScanFromJson(m.Body, t)

		if delay := scan.RateLimit(); delay != 0 {
			go func() {
				time.Sleep(time.Duration(delay) * time.Second)
				err := producer.Publish("seed", scan.Json())
				if err != nil {
					fmt.Println("Could not publish", "fuzz", err)
				}
			}()
		} else {
			// TODO - phantom JS timeout should be an input argument.
			r := &renderer.PhantomJSRenderer{Timeout: 60}
			wq <- true
			scan.CrawlAsync(r)
			go func() {
				if s := <-r.GetRequestBody(); s != nil {
					// fmt.Println("Got request body", s.Request.URL)
					err := producer.Publish("fuzz", s.Json())
					if err != nil {
						fmt.Println("Could not publish", "fuzz", err)
					}
				}
			}()

			go func() {
				isUnique := false
				for s := range r.GetLinks() {
					// do the evaluation once only.
					isUnique = isUnique || scan.IsUnique()
					if isUnique {
						if ok := s.ApplyLinkRules(); ok {
							err := producer.Publish("seed", s.Json())
							if err != nil {
								fmt.Println("Could not publish", "seed", err)
							}
						}
					}
				}
				<-wq
			}()
		}

		return nil
	})

	producer = newProducer()
	defer producer.Stop()
	consumer = newConsumer("seed", "primary", handler)
	defer consumer.Stop()

	wg.Wait()

}
示例#3
0
func fuzzWithArachni() {
	var consumer *nsq.Consumer
	handler := nsq.HandlerFunc(func(m *nsq.Message) error {
		wq <- true
		scan := gryffin.NewScanFromJson(m.Body, t)
		f := &arachni.Fuzzer{}
		f.Fuzz(scan)
		<-wq
		return nil
	})
	consumer = newConsumer("fuzz", "arachni", handler)
	defer consumer.Stop()
	wg.Wait()
}