Ejemplo n.º 1
0
func main() {
	crawler := proxy.NewCrawler([]proxy.Source{proxy.CyberSource}, *timeout, 3)
	recvCh := make(chan *proxy.Proxy, *cap)
	go crawler.FetchProxys(recvCh, *testURL, proxy.DefaultCheck, nil)
	for p := range recvCh {
		fmt.Println(p.String(), " ", p.ConnTime)
	}
}
Ejemplo n.º 2
0
func main() {
	cache := NewProxysCache()
	c := crawler{
		Crawler:   proxy.NewCrawler([]proxy.Source{proxy.CyberSource}, 2*time.Second, 3),
		Observers: []Observer{cache},
		TestURL:   config.TestURL,
		Cap:       config.Cap,
		Check:     proxy.DefaultCheck,
	}

	go func() {
		ticker := time.Tick(config.Interval)
		for {
			cancelCh := make(chan struct{})
			go c.Crawl(cancelCh)
			<-ticker
			close(cancelCh)
		}
	}()

	http.HandleFunc("/proxys", CacheHandler(cache))

	log.Fatal(http.ListenAndServe(":8080", nil))
}