Example #1
0
// It will construct the client and the request, then send it
//
// This function has to be called as the last thing,
// after setting the other properties
func (f *Request) Send() (*http.Response, error) {
	if f.client == nil {
		f.client = http.DefaultClient
	}
	if f.timeout != 0 {
		f.client.Timeout = f.timeout
	}

	if f.proxy != "" {
		proxyUrl, err := url.Parse(f.proxy)
		if err != nil {
			return nil, err
		}

		if f.client.Transport != nil {
			f.client.Transport.(*http.Transport).Proxy = http.ProxyURL(proxyUrl)
		} else {
			f.client.Transport = &http.Transport{
				Proxy: http.ProxyURL(proxyUrl),
			}
		}
	}

	res, err := f.do(f.client)
	return res, err
}
Example #2
0
func (r *runner) setupDiscoveryProxy() error {
	uri, err := url.Parse(r.config.NetworkConfig.ProxyURL)
	if err != nil {
		r.log.Warnf("Failed to parse proxy url: %v", err)
		return nil
	}

	// discovery requests
	transport, ok := discovery.Client.Transport.(*http.Transport)
	if !ok {
		r.log.Warnf("Failed to configure discovery proxy, transport was not the expected type: %T",
			discovery.Client.Transport)
		return nil
	}
	transport.Proxy = http.ProxyURL(uri)

	// actual download requests
	transport, ok = aciremote.Client.Transport.(*http.Transport)
	if !ok {
		r.log.Warnf("Failed to configure remote download proxy, transport was not the expected type: %T",
			aciremote.Client.Transport)
		return nil
	}
	transport.Proxy = http.ProxyURL(uri)

	return nil
}
Example #3
0
// executes the given `Request` object and returns response
func (c *Client) execute(req *Request) (*Response, error) {
	// Apply Request middleware
	var err error
	for _, f := range c.beforeRequest {
		err = f(c, req)
		if err != nil {
			return nil, err
		}
	}

	c.mutex.Lock()

	if req.proxyURL != nil {
		c.transport.Proxy = http.ProxyURL(req.proxyURL)
	} else if c.proxyURL != nil {
		c.transport.Proxy = http.ProxyURL(c.proxyURL)
	} else {
		c.transport.Proxy = nil
	}

	req.Time = time.Now()
	c.httpClient.Transport = c.transport

	resp, err := c.httpClient.Do(req.RawRequest)

	c.mutex.Unlock()

	response := &Response{
		Request:     req,
		RawResponse: resp,
		receivedAt:  time.Now(),
	}

	if err != nil {
		return response, err
	}

	if !req.isSaveResponse {
		defer resp.Body.Close()
		response.body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return response, err
		}

		response.size = int64(len(response.body))
	}

	// Apply Response middleware
	for _, f := range c.afterResponse {
		err = f(c, response)
		if err != nil {
			break
		}
	}

	return response, err
}
Example #4
0
func FetchUrl(theurl string) string {
	var client *http.Client

	if proxy := os.Getenv("http_proxy"); proxy != `` {
		proxyUrl, err := url.Parse(proxy)
		CheckError(err)

		transport := http.Transport{
			Dial:  TimeoutDialer(5*time.Second, 5*time.Second), // connect, read/write
			Proxy: http.ProxyURL(proxyUrl),
		}

		client = &http.Client{Transport: &transport}
	} else {
		client = &http.Client{}
	}

	req, err := http.NewRequest(`GET`, theurl, nil)
	CheckError(err)

	resp, err := client.Do(req)
	CheckError(err)

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	CheckError(err)

	return string(body)
}
Example #5
0
// doRequest makes a request to Chronos REST API
func (cl Client) doRequest(req *http.Request) ([]byte, error) {

	// Init a client
	client := cl.Client

	if cl.ProxyURL != nil {
		client = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(cl.ProxyURL)}}
	} else {
		if cl.Client == nil {
			client = &http.Client{}
		}
	}

	// Do request
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// Read data
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return data, errors.New("bad response: " + fmt.Sprintf("%d", resp.StatusCode))
	}

	return data, nil
}
func main() {
	// CIRCUIT Client
	proxyurl, _ := url.Parse(PROXY_URL)
	circuit_tr := &http.Transport{
		// Ignore SSL errors
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	circuit_client := &http.Client{Transport: circuit_tr}

	// SALESFORCE Client
	salesforce_tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		// Configure proxy
		Proxy: http.ProxyURL(proxyurl),
	}
	salesforce_client := &http.Client{Transport: salesforce_tr}

	// SERVER (Extension App)
	mux := http.NewServeMux()

	// Server Routes
	mux.HandleFunc("/configure", configureHandler)

	// periodically check for leads (polling due to infr. restrictions)
	go poll(salesforce_client, circuit_client)

	// Start server!
	fmt.Println("... listening on " + PORT)
	if err := http.ListenAndServe(PORT, mux); nil != err {
		log.Fatal(err.Error())
	}
}
Example #7
0
// proxyFuncFromEnv builds a proxy function if the appropriate environment
// variable is set. It performs basic sanitization of the environment variable
// and returns any error encountered.
func proxyFuncFromEnv() (func(*http.Request) (*url.URL, error), error) {
	proxy := os.Getenv(DiscoveryProxyEnv)
	if proxy == "" {
		return nil, nil
	}
	// Do a small amount of URL sanitization to help the user
	// Derived from net/http.ProxyFromEnvironment
	proxyURL, err := url.Parse(proxy)
	if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
		// proxy was bogus. Try prepending "http://" to it and
		// see if that parses correctly. If not, we ignore the
		// error and complain about the original one
		var err2 error
		proxyURL, err2 = url.Parse("http://" + proxy)
		if err2 == nil {
			err = nil
		}
	}
	if err != nil {
		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
	}

	log.Printf("discovery: using proxy %q", proxyURL.String())
	return http.ProxyURL(proxyURL), nil
}
Example #8
0
func NewClient(
	esURL, index string, proxyURL *url.URL, tls *tls.Config,
	username, password string,
	params map[string]string,
	onConnectCallback connectCallback,
) *Client {
	proxy := http.ProxyFromEnvironment
	if proxyURL != nil {
		proxy = http.ProxyURL(proxyURL)
	}

	client := &Client{
		Connection: Connection{
			URL:      esURL,
			Username: username,
			Password: password,
			http: &http.Client{
				Transport: &http.Transport{
					TLSClientConfig: tls,
					Proxy:           proxy,
				},
			},
		},
		index:  index,
		params: params,
	}

	client.Connection.onConnectCallback = func() error {
		if onConnectCallback != nil {
			return onConnectCallback(client)
		}
		return nil
	}
	return client
}
Example #9
0
func (b *Boomer) runWorker(n int) {
	var throttle <-chan time.Time
	if b.Qps > 0 {
		throttle = time.Tick(time.Duration(1e6/(b.Qps)) * time.Microsecond)
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
		DisableCompression: b.DisableCompression,
		DisableKeepAlives:  b.DisableKeepAlives,
		// TODO(jbd): Add dial timeout.
		TLSHandshakeTimeout: time.Duration(b.Timeout) * time.Millisecond,
		Proxy:               http.ProxyURL(b.ProxyAddr),
	}
	if b.H2 {
		http2.ConfigureTransport(tr)
	} else {
		tr.TLSNextProto = make(map[string]func(string, *tls.Conn) http.RoundTripper)
	}
	client := &http.Client{Transport: tr}
	for i := 0; i < n; i++ {
		if b.Qps > 0 {
			<-throttle
		}
		b.makeRequest(client)
	}
}
func NewProxyAuthTransport(rawTransport *http.Transport, customHeaders http.Header) (*ProxyAuthTransport, error) {
	dialFn := rawTransport.Dial
	if dialFn == nil {
		dialFn = net.Dial
	}
	tr := &ProxyAuthTransport{Dial: dialFn, CustomHeaders: customHeaders}
	proxyUrlFn := rawTransport.Proxy
	if proxyUrlFn != nil {
		wrappedDialFn := tr.wrapTransportDial()
		rawTransport.Dial = wrappedDialFn
		proxyUrl, err := proxyUrlFn(nil)
		if err != nil {
			return nil, err
		}
		if proxyUrl.Scheme != "http" {
			return nil, fmt.Errorf("Only HTTP proxy supported, for SOCKS use http.Transport with custom dialers & upstreamproxy.NewProxyDialFunc")
		}
		if proxyUrl.User != nil {
			tr.Username = proxyUrl.User.Username()
			tr.Password, _ = proxyUrl.User.Password()
		}
		// strip username and password from the proxyURL because
		// we do not want the wrapped transport to handle authentication
		proxyUrl.User = nil
		rawTransport.Proxy = http.ProxyURL(proxyUrl)
	}

	tr.Transport = rawTransport
	return tr, nil
}
func CheckAnonyPost(proxyAddr string) (bool, float64) { //验证Post方法
	start := time.Now()
	proxy, err1 := url.Parse(proxyAddr)
	if err1 != nil {
		// log.Println(err1)
		return false, 0
	}
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(10 * time.Second)
				c, err2 := net.DialTimeout(netw, addr, 10*time.Second)
				if err2 != nil {
					return nil, err2
				}
				c.SetDeadline(deadline)
				return c, nil
			},
			DisableKeepAlives:     true,
			ResponseHeaderTimeout: 10 * time.Second,
			DisableCompression:    false,
			Proxy:                 http.ProxyURL(proxy),
		},
	}
	respPost, err4 := client.Post(PROXY_CHECKER_SERVICE, "application/x-www-form-urlencoded", strings.NewReader("username=credit")) //验证Post方法
	if err4 != nil {
		// log.Println(err4)
		return false, 0
	}
	bodyPost, _ := ioutil.ReadAll(respPost.Body)
	defer respPost.Body.Close()
	return respPost.StatusCode == http.StatusOK && string(bodyPost) == "ok", time.Now().Sub(start).Seconds()
}
Example #12
0
func checkProxy(proxy string, downloadedUrl string) (success bool, errorMessage string) {
	getsInProgress <- 1
	defer func() { <-getsInProgress }()
	if !strings.HasPrefix(proxy, "http") {
		proxy = "http://" + proxy
	}
	proxyUrl, err := url.Parse(proxy)
	httpClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
	response, err := httpClient.Get(downloadedUrl)
	if err != nil {
		return false, err.Error()
	}

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return false, err.Error()
	}

	bodyString := strings.ToLower(strings.Trim(string(body), " \n\t\r"))

	if strings.Index(bodyString, "<body") < 0 && strings.Index(bodyString, "<head") < 0 {
		if strings.Index(bodyString, "<title>invalid request</title>") >= 0 {
			return false, "Tracker responsed 'Invalid request' - might be dead"
		} else {
			return false, "Reveived page is not HTML: " + bodyString
		}
	}

	return true, ""
}
Example #13
0
func httpProxyClient(ip string, block *Block) *http.Client {
	proxy, _ := url.Parse("http://" + ip)
	transport := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			timeout := time.Duration(10) * time.Second
			deadline := time.Now().Add(timeout)
			c, err := net.DialTimeout(netw, addr, timeout)
			if err != nil {
				return nil, err
			}
			c.SetDeadline(deadline)
			return c, nil
		},
		Proxy: http.ProxyURL(proxy),
		ResponseHeaderTimeout: time.Second * 10,
	}

	client := &http.Client{
		Transport: transport,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			log.Println("redirect to:", req.URL.String(), ip)
			block.block(ip)
			return errors.New("does not allow redirect")
		},
	}
	return client
}
Example #14
0
func BuildHTTPClient(proxy, token string) *http.Client {
	var base http.RoundTripper
	base = http.DefaultTransport

	// Proxy layer
	if len(proxy) > 0 {
		u, _ := url.Parse(proxy)
		base = &http.Transport{
			Proxy: http.ProxyURL(u),
		}
	}

	// Authentication layer
	if len(token) > 0 {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		base = &oauth2.Transport{
			Source: ts,
			Base:   base,
		}
	}

	// Rate limiting
	transport := &RateLimitedTransport{
		Base: base,
	}

	return &http.Client{
		Transport: transport,
	}
}
Example #15
0
func TestClient(t *testing.T) {
	ts := httptest.NewServer(
		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("ok"))
		}),
	)
	defer ts.Close()
	proxy, _ := url.Parse(ts.URL)

	tr := &http.Transport{Proxy: http.ProxyURL(proxy)}
	c := &http.Client{Transport: tr}
	r := New()
	r.Client(c)
	res, err := r.Get("http://github.com").Send()
	if err != nil {
		t.Error(err)
	}

	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		t.Error(err)
	}
	if string(body) != "ok" {
		t.Error("Incorrect response.")
	}
}
Example #16
0
File: gcm.go Project: pkar/hermes
// NewGCMClient ...
func NewGCMClient(apiURL, key, proxy string) (*GCMClient, error) {
	if apiURL == "" {
		return nil, fmt.Errorf("url not provided")
	}
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			deadline := time.Now().Add(20 * time.Second)
			c, err := net.DialTimeout(netw, addr, time.Second*20)
			if err != nil {
				return nil, err
			}
			c.SetDeadline(deadline)
			return c, nil
		},
		DisableKeepAlives: true,
	}
	if proxy != "" {
		// http://proxyIp:proxyPort
		proxyUrl, err := url.Parse(proxy)
		if err != nil {
			return nil, err
		}
		tr.Proxy = http.ProxyURL(proxyUrl)
	}

	return &GCMClient{
		key:  key,
		http: &http.Client{Transport: tr},
		url:  apiURL,
	}, nil
}
Example #17
0
// init
func (c *Hammer) Init() {
	c.counter = new(scenario.Counter)
	// c.counterArray = make([][]int64, 0)
	// set up HTTP proxy
	if proxy != "none" {
		proxyUrl, err := url.Parse(proxy)
		if err != nil {
			log.Fatal(err)
		}
		c.client = &http.Client{
			Transport: &http.Transport{
				// DisableKeepAlives:   false,
				// MaxIdleConnsPerHost: 200000,
				Proxy: http.ProxyURL(proxyUrl),
			},
		}
	} else {
		c.client = &http.Client{
			Transport: &http.Transport{
			// DisableKeepAlives:   false,
			// MaxIdleConnsPerHost: 200000,
			},
		}
	}
}
Example #18
0
func NewClient(
	esURL, index string, proxyURL *url.URL, tls *tls.Config,
	username, password string,
) *Client {
	proxy := http.ProxyFromEnvironment
	if proxyURL != nil {
		proxy = http.ProxyURL(proxyURL)
	}

	client := &Client{
		Connection{
			URL:      esURL,
			Username: username,
			Password: password,
			http: &http.Client{
				Transport: &http.Transport{
					TLSClientConfig: tls,
					Proxy:           proxy,
				},
			},
		},
		index,
	}
	return client
}
Example #19
0
func (this *ReqQiushiModule) Init(global_conf *context.GlobalContext) (err error) {
	this.qiushi_url = global_conf.Qiushi.Location
	this.timeout = global_conf.Qiushi.Timeout
	/*********设置传输层参数****************/
	transport := &http.Transport{}
	transport.Dial = func(netw, addr string) (net.Conn, error) {
		c, err := net.DialTimeout(netw, addr, time.Millisecond*time.Duration(this.timeout))
		if err != nil {
			utils.WarningLog.Write("dail timeout [%s]", err.Error())
			return nil, err
		}
		return c, nil
	}
	transport.MaxIdleConnsPerHost = 10
	transport.ResponseHeaderTimeout = time.Millisecond * time.Duration(this.timeout)
	if global_conf.Proxy.Open {
		url_i := url.URL{}
		url_proxy, _ := url_i.Parse(global_conf.Proxy.Location)
		transport.Proxy = http.ProxyURL(url_proxy)
		utils.DebugLog.Write("open http proxy , proxy location [%s]", global_conf.Proxy.Location)
	}
	/**********************************/
	this.client = &http.Client{}
	this.client.Transport = transport
	utils.DebugLog.Write("req qiushi url [%s]", this.qiushi_url)

	return
}
Example #20
0
func applyProxy(a *Args) (err error) {
	if a.Proxy == "" {
		return nil
	}

	u, err := url.Parse(a.Proxy)
	if err != nil {
		return err
	}
	switch u.Scheme {
	case "http", "https":
		a.Client.Transport = &http.Transport{
			Proxy: http.ProxyURL(u),
			Dial: (&net.Dialer{
				Timeout: 30 * time.Second,
				// KeepAlive: 30 * time.Second,
			}).Dial,
			// TLSHandshakeTimeout: 10 * time.Second,
		}
	case "socks5":
		dialer, err := proxy.FromURL(u, proxy.Direct)
		if err != nil {
			return err
		}
		a.Client.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  dialer.Dial,
			// TLSHandshakeTimeout: 10 * time.Second,
		}
	}
	return
}
Example #21
0
func main() {
	// Before running this, be sure to install Polipo: 'sudo apt-get
	// install polipo'. Get the config file at
	// https://gitweb.torproject.org/torbrowser.git/blob_plain/1ffcd9dafb9dd76c3a29dd686e05a71a95599fb5:/build-scripts/config/polipo.conf
	// then save it to /etc/polipo/config and restart Polipo with
	// 'sudo service polipo restart', _then_ run this program.

	go noProxy()

	proxyURL, err := url.Parse(PROXY_URL)
	fun.MaybeFatalAt("url.Parse", err)

	transport := &http.Transport{Proxy: http.ProxyURL(proxyURL)}
	// fmt.Printf("transport == %+v\n", transport)
	client := &http.Client{Transport: transport}

	request, err := http.NewRequest("GET", REMOTE_HTTP_ADDR, nil)
	fun.MaybeFatalAt("NewRequest", err)

	resp, err := client.Do(request)
	fun.MaybeFatalAt("client.Do", err)

	// resp, err := http.Get(REMOTE_HTTP_ADDR)
	// fun.MaybeFatalAt("Get", err)
	body, err := ioutil.ReadAll(resp.Body)
	fun.MaybeFatalAt("ReadAll", err)
	defer resp.Body.Close()

	fmt.Printf("Torified: %s", body)
}
func prepares() {
	checkConfig()
	proxyUrl, _ := url.Parse("http://" + config.ProxyHost + ":" + config.ProxyPort)
	http.DefaultTransport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
	checkAndMakeDir(resultsDir)
	if config.DetailedData {
		checkAndMakeDir(resultsDir + "/" + detailedDir)
		checkAndMakeDir(resultsDir + "/" + detailedDir + "/" + accountDir)
		checkAndMakeDir(resultsDir + "/" + detailedDir + "/" + hashtagDir)
	}
	checkAndMakeDir(resultsDir + "/" + sortedDir)
	checkAndMakeDir(resultsDir + "/" + unsortedDir)
	hashtagsByAccount = make(map[string]map[string]struct{})
	totalHashtags = make(map[string]struct{})
	externalLinksByAccount = make(map[string]map[string]struct{})
	totalExternalLinks = make(map[string]struct{})
	userAccountsByAccount = make(map[string]map[string]struct{})
	totalUserAccounts = make(map[string]struct{})
	accountsToSearch = make(map[string]struct{})
	hashtagsToSearch = make(map[string]struct{})
	findedAccountsToSearch = make(map[string]struct{})
	userAccountsByHashtag = make(map[string]map[string]struct{})
	externalLinksByHashtag = make(map[string]map[string]struct{})
	hashtagsByHashtag = make(map[string]map[string]struct{})
	loadHashtags()
	loadAccounts()
}
Example #23
0
func post_test_proxy() {
	post_url := "http://192.168.7.82/portal.jpad/authenticate.php"
	req, err := http.NewRequest("POST", post_url, strings.NewReader("server=192.168.7.82&macSerial=0081050129EA&login=zsy&password=lTXhvSj4mzk%3D&version=test_160511&mobile=0"))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	proxyUrl, err := url.Parse("http://127.0.0.1:8888")
	httpClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
	resp, err := httpClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	// body, _ := ioutil.ReadAll(resp.Body)
	// fmt.Println("response Body:", string(body))
	// fmt.Println("\u554a")
	// json
	decoder := json.NewDecoder(resp.Body)
	var data Authenticate_info
	err = decoder.Decode(&data)
	if err != nil {
		log.Fatal(err)
		return
	}
	fmt.Println(data.Message)
}
Example #24
0
func (t TransportGet) GetByProxy() {
	req, err := http.NewRequest("GET", t.url, nil)
	checkError("NewRequest", err)
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11")

	proxyURL, err := url.Parse(t.addr)
	checkError("Parse proxy", err)
	client := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyURL(proxyURL),
		},
		Timeout: 3 * time.Second,
	}

	resp, err := client.Do(req)
	if err != nil {
		// log.Printf("Request do: %v\n", err)
		fmt.Printf("代理服务器[%s] 访问url[%s]失败.\n", t.addr, t.url)
		return
	}
	if resp.StatusCode == http.StatusOK {
		fmt.Printf("代理服务器[%s] 访问url[%s]成功.\n", t.addr, t.url)
	} else {
		fmt.Printf("代理服务器[%s] 访问url[%s]失败.\n", t.addr, t.url)
	}

	defer resp.Body.Close()
	time.Sleep(10 * time.Second)
}
Example #25
0
// SetDownstreamProxy sets the proxy that receives requests from the upstream
// proxy.
func (p *Proxy) SetDownstreamProxy(proxyURL *url.URL) {
	p.proxyURL = proxyURL

	if tr, ok := p.roundTripper.(*http.Transport); ok {
		tr.Proxy = http.ProxyURL(p.proxyURL)
	}
}
Example #26
0
// buildClient creates, configures, and returns a *http.Client type.
func (self *Surf) buildClient(param *Param) *http.Client {
	client := &http.Client{
		CheckRedirect: param.checkRedirect,
	}

	if param.enableCookie {
		client.Jar = self.cookieJar
	}

	transport := &http.Transport{
		Dial: func(network, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(network, addr, param.dialTimeout)
			if err != nil {
				return nil, err
			}
			if param.connTimeout > 0 {
				c.SetDeadline(time.Now().Add(param.connTimeout))
			}
			return c, nil
		},
	}

	if param.proxy != nil {
		transport.Proxy = http.ProxyURL(param.proxy)
	}

	if strings.ToLower(param.url.Scheme) == "https" {
		transport.TLSClientConfig = &tls.Config{RootCAs: nil, InsecureSkipVerify: true}
		transport.DisableCompression = true
	}
	client.Transport = transport
	return client
}
Example #27
0
// newHttpClient returns a new http client that is configured according to the
// proxy and TLS settings in the associated connection configuration.
func newHttpClient(config *ConnConfig) (*http.Client, error) {
	// Set proxy function if there is a proxy configured.
	var proxyFunc func(*http.Request) (*url.URL, error)
	if config.Proxy != "" {
		proxyURL, err := url.Parse(config.Proxy)
		if err != nil {
			return nil, err
		}
		proxyFunc = http.ProxyURL(proxyURL)
	}

	// Configure TLS if needed.
	var tlsConfig *tls.Config
	if !config.DisableTLS {
		pool := x509.NewCertPool()
		pool.AppendCertsFromPEM(config.Certificates)
		tlsConfig = &tls.Config{
			RootCAs: pool,
		}
	}

	client := http.Client{
		Transport: &http.Transport{
			Proxy:           proxyFunc,
			TLSClientConfig: tlsConfig,
		},
	}

	return &client, nil
}
Example #28
0
// Proxy function accepts a proxy url string to setup proxy url for any request.
// It provides a convenience way to setup proxy which have advantages over usual old ways.
// One example is you might try to set `http_proxy` environment. This means you are setting proxy up for all the requests.
// You will not be able to send different request with different proxy unless you change your `http_proxy` environment again.
// Another example is using Golang proxy setting. This is normal prefer way to do but too verbase compared to GoRequest's Proxy:
//
//      gorequest.New().Proxy("http://myproxy:9999").
//        Post("http://www.google.com").
//        End()
//
// To set no_proxy, just put empty string to Proxy func:
//
//      gorequest.New().Proxy("").
//        Post("http://www.google.com").
//        End()
//
func (s *SuperAgent) Proxy(proxyUrl string) *SuperAgent {
	parsedProxyUrl, err := url.Parse(proxyUrl)
	if err != nil {
		s.Errors = append(s.Errors, err)
	} else if proxyUrl == "" {
		s.Transport.Proxy = nil
	} else if parsedProxyUrl.Scheme == "http" || parsedProxyUrl.Scheme == "https" {
		s.Transport.Proxy = http.ProxyURL(parsedProxyUrl)
		if parsedProxyUrl.User != nil {
			user := parsedProxyUrl.User.Username()
			pwd, _ := parsedProxyUrl.User.Password()
			s = s.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(user+":"+pwd)))
		}
	} else if parsedProxyUrl.Scheme == "socks5" {
		dialer, err := proxy.FromURL(parsedProxyUrl, s)
		if err != nil {
			s.Errors = append(s.Errors, err)
		} else {
			s.Transport.Dial = dialer.Dial
		}
	} else {
		s.Errors = append(s.Errors, errors.New("proxy: unknown scheme: "+parsedProxyUrl.Scheme))
	}
	return s
}
Example #29
0
func makeRequest(requestType string, url string, reqBody []byte) (*http.Response, []byte, error) {
	req, err := http.NewRequest(requestType, Config.APIUrl+url, bytes.NewBuffer(reqBody))

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Cachet-Token", Config.APIToken)

	client := &http.Client{}
	transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
	if Config.InsecureAPI == true {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}
	if Config.Proxy != "" {
		proxyUrl, err := urlparser.Parse(Config.Proxy)
		if err != nil {
			return nil, []byte{}, err
		}
		transport.Proxy = http.ProxyURL(proxyUrl)
	}
	client.Transport = transport

	res, err := client.Do(req)
	if err != nil {
		return nil, []byte{}, err
	}

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	return res, body, nil
}
Example #30
0
func LoadImageFromURL(link string, proxy string, pageLink string) image.Image {
	req, err := http.NewRequest("GET", link, nil)
	if err != nil || req == nil || req.Header == nil {
		return nil
	}
	client := &http.Client{}
	if proxy != "" {
		proxyUrl, _ := url.Parse(proxy)
		client = &http.Client{
			Transport: &http.Transport{
				Dial:                  dialTimeout,
				DisableKeepAlives:     true,
				ResponseHeaderTimeout: 10 * time.Second,
				Proxy: http.ProxyURL(proxyUrl),
			},
		}
	}
	req.Header.Add("Referer", pageLink)
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0")
	resp, err := client.Do(req)

	if err != nil || resp == nil || resp.Body == nil {
		return nil
	} else {
		defer resp.Body.Close()
		img, _, err := image.Decode(resp.Body)
		if err != nil {
			panic(err)
		}
		return img
	}
}