Example #1
0
func (this *Engine) SetConfig(config *common.Config) *Engine {
	this.config = config
	this.resourceManager = common.NewResourceManager(config.GetConcurrency())
	if !this.config.GetLogging() {
		log.SetOutput(ioutil.Discard)
	}
	return this
}
func (this *HttpDownloader) Download(req *common.Request, config *common.Config) (*common.Response, error) {
	for key, value := range config.GetHeaders() {
		req.Request.Header.Set(key, value)
	}

	transport := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, config.GetConnectionTimeout())
			if err != nil {
				return nil, err
			}
			return c, nil
		},
		ResponseHeaderTimeout: config.GetDownloadTimeout(),
		MaxIdleConnsPerHost:   config.GetMaxIdleConnsPerHost(),
	}
	client := &http.Client{
		Timeout:   2 * config.GetDownloadTimeout(),
		Transport: transport,
	}
	if req.ProxyUrl != "" {
		transport.Proxy = http.ProxyURL(&url.URL{Host: req.ProxyUrl})
	}
	if req.Jar != nil {
		client.Jar = req.Jar
	}
	if req.Error != nil {
		return common.NewResponse(nil, req.Url, ""), req.Error
	}

	resp, err := common.NewCurl(client, req).Do()
	if err != nil {
		log.Printf("curl %s error %s\n", req.Url, err)
		return resp, err
	}

	if config.GetSucc() != "" && !strings.Contains(resp.Body, config.GetSucc()) {
		return resp, errors.New(fmt.Sprintf("Invalid response body(succ: %s)", config.GetSucc()))
	}
	return resp, nil
}