func (this *HttpDownloader) Download(req *common.Request, config *common.Config) (*common.Response, error) {
for key, value := range config.GetHeaders() {
req.Request.Header.Set(key, value)
}
transport := &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
c, err := net.DialTimeout(netw, addr, config.GetConnectionTimeout())
if err != nil {
return nil, err
}
return c, nil
},
ResponseHeaderTimeout: config.GetDownloadTimeout(),
MaxIdleConnsPerHost: config.GetMaxIdleConnsPerHost(),
}
client := &http.Client{
Timeout: 2 * config.GetDownloadTimeout(),
Transport: transport,
}
if req.ProxyUrl != "" {
transport.Proxy = http.ProxyURL(&url.URL{Host: req.ProxyUrl})
}
if req.Jar != nil {
client.Jar = req.Jar
}
if req.Error != nil {
return common.NewResponse(nil, req.Url, ""), req.Error
}
resp, err := common.NewCurl(client, req).Do()
if err != nil {
log.Printf("curl %s error %s\n", req.Url, err)
return resp, err
}
if config.GetSucc() != "" && !strings.Contains(resp.Body, config.GetSucc()) {
return resp, errors.New(fmt.Sprintf("Invalid response body(succ: %s)", config.GetSucc()))
}
return resp, nil
}
func GetCookieFunc(req *common.Request) (*cookiejar.Jar, error) {
if _, ok := gAuth.IsAuthed[req.ProxyUrl]; ok {
log.Printf("have authed %+v\n", gAuth.Jar[req.ProxyUrl])
return gAuth.Jar[req.ProxyUrl], nil
}
baseUrl := "http://bgp.he.net"
transport := &http.Transport{
Proxy: http.ProxyURL(&url.URL{Host: req.ProxyUrl}),
Dial: func(netw, addr string) (net.Conn, error) {
c, err := net.DialTimeout(netw, addr, gConfig.GetConnectionTimeout())
if err != nil {
return nil, err
}
return c, nil
},
ResponseHeaderTimeout: gConfig.GetDownloadTimeout(),
MaxIdleConnsPerHost: gConfig.GetMaxIdleConnsPerHost(),
}
gAuth.Jar[req.ProxyUrl], _ = cookiejar.New(nil)
client := &http.Client{
Jar: gAuth.Jar[req.ProxyUrl],
Timeout: 2 * gConfig.GetDownloadTimeout(),
Transport: transport,
}
var p string
var i string
{
u := baseUrl + "/i"
resp, err := common.NewCurl(client, common.NewRequest(u)).Do()
if err != nil {
log.Printf("1. auth failed(%s) %s\n", u, err)
return nil, err
}
i = strings.Trim(resp.Response.Header.Get("ETag"), "\"")
}
{
u := baseUrl + "/dns/qq.com"
_, err := common.NewCurl(client, common.NewRequest(u)).Do()
if err != nil {
log.Printf("2. auth failed(%s) %s\n", u, err)
return nil, err
}
path := ""
for _, c := range gAuth.Jar[req.ProxyUrl].Cookies(req.Request.URL) {
if c.Name == "path" {
path = c.Value
break
}
}
decodedPath, _ := url.QueryUnescape(path)
p = fmt.Sprintf("%x", md5.Sum([]byte(decodedPath)))
}
{
u := baseUrl + "/cc"
_, err := common.NewCurl(client, common.NewRequest(u)).Do()
if err != nil {
log.Printf("3. auth failed(%s) %s\n", u, err)
return nil, err
}
}
{
u := baseUrl + "/jc"
form := url.Values{}
form.Add("p", p)
form.Add("i", i)
r := common.NewRequest(u)
r.Request, _ = http.NewRequest("POST", u, strings.NewReader(form.Encode()))
_, err := common.NewCurl(client, r).Do()
if err != nil {
log.Printf("4.auth failed(%s) %s\n", u, err)
return nil, err
}
}
gAuth.IsAuthed[req.ProxyUrl] = true
log.Printf("auth succeed %+v\n", gAuth.Jar[req.ProxyUrl])
return gAuth.Jar[req.ProxyUrl], nil
}