func New() *Proxy { return (&Proxy{ ipRegexp: regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+`), proxyRegexp: regexp.MustCompile(`http[s]?://[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+`), allIps: map[string]string{}, all: map[string]bool{}, usable: make(map[string]*ProxyForHost), threadPool: make(chan bool, MAX_THREAD_NUM), surf: surfer.New(), }).Update() }
"github.com/henrylee2cn/pholcus/app/downloader/surfer" "github.com/henrylee2cn/pholcus/config" ) const ( SURF_ID = 0 //默认下载器,此值不可改动 PHANTOM_ID = iota ) type Surfer struct { surf surfer.Surfer phantom surfer.Surfer } var SurferDownloader = &Surfer{ surf: surfer.New(), phantom: surfer.NewPhantom(config.SURFER_PHANTOM.FULL_APP_NAME, config.SURFER_PHANTOM.FULL_TEMP_JS), } func (self *Surfer) Download(cReq *context.Request) *context.Response { cResp := context.NewResponse(nil) var resp *http.Response var err error switch cReq.GetDownloaderID() { case SURF_ID: resp, err = self.surf.Download(cReq) case PHANTOM_ID: resp, err = self.phantom.Download(cReq)