func main() { crawler := proxy.NewCrawler([]proxy.Source{proxy.CyberSource}, *timeout, 3) recvCh := make(chan *proxy.Proxy, *cap) go crawler.FetchProxys(recvCh, *testURL, proxy.DefaultCheck, nil) for p := range recvCh { fmt.Println(p.String(), " ", p.ConnTime) } }
func main() { cache := NewProxysCache() c := crawler{ Crawler: proxy.NewCrawler([]proxy.Source{proxy.CyberSource}, 2*time.Second, 3), Observers: []Observer{cache}, TestURL: config.TestURL, Cap: config.Cap, Check: proxy.DefaultCheck, } go func() { ticker := time.Tick(config.Interval) for { cancelCh := make(chan struct{}) go c.Crawl(cancelCh) <-ticker close(cancelCh) } }() http.HandleFunc("/proxys", CacheHandler(cache)) log.Fatal(http.ListenAndServe(":8080", nil)) }