func (this *HttpDownloader) Download(req *common.Request, config *common.Config) (*common.Response, error) {
	for key, value := range config.GetHeaders() {
		req.Request.Header.Set(key, value)
	}

	transport := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, config.GetConnectionTimeout())
			if err != nil {
				return nil, err
			}
			return c, nil
		},
		ResponseHeaderTimeout: config.GetDownloadTimeout(),
		MaxIdleConnsPerHost:   config.GetMaxIdleConnsPerHost(),
	}
	client := &http.Client{
		Timeout:   2 * config.GetDownloadTimeout(),
		Transport: transport,
	}
	if req.ProxyUrl != "" {
		transport.Proxy = http.ProxyURL(&url.URL{Host: req.ProxyUrl})
	}
	if req.Jar != nil {
		client.Jar = req.Jar
	}
	if req.Error != nil {
		return common.NewResponse(nil, req.Url, ""), req.Error
	}

	resp, err := common.NewCurl(client, req).Do()
	if err != nil {
		log.Printf("curl %s error %s\n", req.Url, err)
		return resp, err
	}

	if config.GetSucc() != "" && !strings.Contains(resp.Body, config.GetSucc()) {
		return resp, errors.New(fmt.Sprintf("Invalid response body(succ: %s)", config.GetSucc()))
	}
	return resp, nil
}
Example #2
0
func GetCookieFunc(req *common.Request) (*cookiejar.Jar, error) {
	if _, ok := gAuth.IsAuthed[req.ProxyUrl]; ok {
		log.Printf("have authed %+v\n", gAuth.Jar[req.ProxyUrl])
		return gAuth.Jar[req.ProxyUrl], nil
	}

	baseUrl := "http://bgp.he.net"
	transport := &http.Transport{
		Proxy: http.ProxyURL(&url.URL{Host: req.ProxyUrl}),
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, gConfig.GetConnectionTimeout())
			if err != nil {
				return nil, err
			}
			return c, nil
		},
		ResponseHeaderTimeout: gConfig.GetDownloadTimeout(),
		MaxIdleConnsPerHost:   gConfig.GetMaxIdleConnsPerHost(),
	}
	gAuth.Jar[req.ProxyUrl], _ = cookiejar.New(nil)
	client := &http.Client{
		Jar:       gAuth.Jar[req.ProxyUrl],
		Timeout:   2 * gConfig.GetDownloadTimeout(),
		Transport: transport,
	}

	var p string
	var i string
	{
		u := baseUrl + "/i"
		resp, err := common.NewCurl(client, common.NewRequest(u)).Do()
		if err != nil {
			log.Printf("1. auth failed(%s) %s\n", u, err)
			return nil, err
		}
		i = strings.Trim(resp.Response.Header.Get("ETag"), "\"")
	}
	{
		u := baseUrl + "/dns/qq.com"
		_, err := common.NewCurl(client, common.NewRequest(u)).Do()
		if err != nil {
			log.Printf("2. auth failed(%s) %s\n", u, err)
			return nil, err
		}
		path := ""
		for _, c := range gAuth.Jar[req.ProxyUrl].Cookies(req.Request.URL) {
			if c.Name == "path" {
				path = c.Value
				break
			}
		}
		decodedPath, _ := url.QueryUnescape(path)
		p = fmt.Sprintf("%x", md5.Sum([]byte(decodedPath)))
	}
	{
		u := baseUrl + "/cc"
		_, err := common.NewCurl(client, common.NewRequest(u)).Do()
		if err != nil {
			log.Printf("3. auth failed(%s) %s\n", u, err)
			return nil, err
		}
	}
	{
		u := baseUrl + "/jc"
		form := url.Values{}
		form.Add("p", p)
		form.Add("i", i)
		r := common.NewRequest(u)
		r.Request, _ = http.NewRequest("POST", u, strings.NewReader(form.Encode()))
		_, err := common.NewCurl(client, r).Do()
		if err != nil {
			log.Printf("4.auth failed(%s) %s\n", u, err)
			return nil, err
		}
	}
	gAuth.IsAuthed[req.ProxyUrl] = true
	log.Printf("auth succeed %+v\n", gAuth.Jar[req.ProxyUrl])
	return gAuth.Jar[req.ProxyUrl], nil
}