func New() *Proxy {
return (&Proxy{
ipRegexp: regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+`),
proxyRegexp: regexp.MustCompile(`http[s]?://[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+`),
allIps: map[string]string{},
all: map[string]bool{},
usable: make(map[string]*ProxyForHost),
threadPool: make(chan bool, MAX_THREAD_NUM),
surf: surfer.New(),
}).Update()
}
"github.com/henrylee2cn/pholcus/app/downloader/surfer"
"github.com/henrylee2cn/pholcus/config"
)
const (
SURF_ID = 0 //默认下载器，此值不可改动
PHANTOM_ID = iota
)
type Surfer struct {
surf surfer.Surfer
phantom surfer.Surfer
}
var SurferDownloader = &Surfer{
surf: surfer.New(),
phantom: surfer.NewPhantom(config.SURFER_PHANTOM.FULL_APP_NAME, config.SURFER_PHANTOM.FULL_TEMP_JS),
}
func (self *Surfer) Download(cReq *context.Request) *context.Response {
cResp := context.NewResponse(nil)
var resp *http.Response
var err error
switch cReq.GetDownloaderID() {
case SURF_ID:
resp, err = self.surf.Download(cReq)
case PHANTOM_ID:
resp, err = self.phantom.Download(cReq)