示例#1
1
// proxySettings will default to the default proxy settings if none are provided
// if settings are provided – they will override the environment variables
func (ro RequestOptions) proxySettings(req *http.Request) (*url.URL, error) {
	// No proxies – lets use the default
	if len(ro.Proxies) == 0 {
		return http.ProxyFromEnvironment(req)
	}

	// There was a proxy specified – do we support the protocol?
	if _, ok := ro.Proxies[req.URL.Scheme]; ok {
		return ro.Proxies[req.URL.Scheme], nil
	}

	// Proxies were specified but not for any protocol that we use
	return http.ProxyFromEnvironment(req)

}
示例#2
0
// Creates a new Twitter client with the supplied OAuth configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
//     export HTTP_PROXY=http://localhost:8888
func NewClient(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client {
	var (
		host      = "api.twitter.com"
		base      = "https://" + host
		req, _    = http.NewRequest("GET", "https://api.twitter.com", nil)
		proxy, _  = http.ProxyFromEnvironment(req)
		transport *http.Transport
	)
	if proxy != nil {
		transport = &http.Transport{
			Proxy: http.ProxyURL(proxy),
		}
	} else {
		transport = &http.Transport{}
	}
	return &Client{
		Host: host,
		HttpClient: &http.Client{
			Transport: transport,
		},
		User:     user,
		AppToken: nil,
		OAuth: &oauth1a.Service{
			RequestURL:   base + "/oauth/request_token",
			AuthorizeURL: base + "/oauth/authorize",
			AccessURL:    base + "/oauth/access_token",
			ClientConfig: config,
			Signer:       new(oauth1a.HmacSha1Signer),
		},
	}
}
示例#3
0
// NewClient creates a new Client object.
// The token can be empty if you plan on creating
// the token using the `CreateToken` func. If you want
// to use your existing token, you need to pass it as the paramter.
// It returns a Client object that is pre-configured for usage.
func NewClient(token string) *Client {

	// Perhaps we should allow people to configure this through some
	// variable. It would make the package almost 100% backwards compatible
	// with the public api (https://developer.kwikdesk.com)
	var (
		host      = "platform.kwikdesk.com"
		base      = "https://" + host
		req, _    = http.NewRequest("GET", base, nil)
		proxy, _  = http.ProxyFromEnvironment(req)
		transport *http.Transport
	)

	transport = &http.Transport{}
	if proxy != nil {
		transport = &http.Transport{
			Proxy: http.ProxyURL(proxy),
		}
	}

	return &Client{
		Host:     host,
		FullHost: base,
		HttpClient: &http.Client{
			Transport: transport,
		},
		ContentType: kdHTTPContentType,
		XToken:      token,
	}
}
示例#4
0
// Creates a new Infoblox client with the supplied user/pass configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
//     export HTTP_PROXY=http://localhost:8888
//
// When using a proxy, disable TLS certificate verification with the following:
//    sslVerify = false
func NewClient(host, username, password string, sslVerify bool) *Client {
	var (
		req, _    = http.NewRequest("GET", host, nil)
		proxy, _  = http.ProxyFromEnvironment(req)
		transport *http.Transport
		tlsconfig *tls.Config
	)
	tlsconfig = &tls.Config{
		InsecureSkipVerify: !sslVerify,
	}
	if tlsconfig.InsecureSkipVerify {
		fmt.Println("WARNING: SSL cert verification  disabled!")
	}
	transport = &http.Transport{
		TLSClientConfig: tlsconfig,
	}
	if proxy != nil {
		transport.Proxy = http.ProxyURL(proxy)
	}
	return &Client{
		Host: host,
		HttpClient: &http.Client{
			Transport: transport,
		},
		Username: username,
		Password: password,
	}
}
示例#5
0
func getProxy() *url.URL {
	req, err := http.NewRequest("GET", "https://api.heroku.com", nil)
	PrintError(err, false)
	proxy, err := http.ProxyFromEnvironment(req)
	PrintError(err, false)
	return proxy
}
示例#6
0
// NewMultipleHostReverseProxy creates a reverse proxy that will randomly
// select a host from the passed `targets`
func NewMultipleHostReverseProxy(targets []*url.URL) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		println("CALLING DIRECTOR")
		target := targets[rand.Int()%len(targets)]
		req.URL.Scheme = target.Scheme
		req.URL.Host = target.Host
		req.URL.Path = target.Path
	}
	return &httputil.ReverseProxy{
		Director: director,
		Transport: &http.Transport{
			Proxy: func(req *http.Request) (*url.URL, error) {
				println("CALLING PROXY")
				return http.ProxyFromEnvironment(req)
			},
			Dial: func(network, addr string) (net.Conn, error) {
				println("CALLING DIAL")
				conn, err := (&net.Dialer{
					Timeout:   30 * time.Second,
					KeepAlive: 30 * time.Second,
				}).Dial(network, addr)
				if err != nil {
					println("Error during DIAL:", err.Error())
				}
				return conn, err
			},
			TLSHandshakeTimeout: 10 * time.Second,
		},
	}
}
示例#7
0
文件: client.go 项目: hickeng/govmomi
func (c *Client) SetCertificate(cert tls.Certificate) {
	t := c.Client.Transport.(*http.Transport)

	// Extension certificate
	t.TLSClientConfig.Certificates = []tls.Certificate{cert}

	// Proxy to vCenter host on port 80
	host, _ := splitHostPort(c.u.Host)

	// Should be no reason to change the default port other than testing
	port := os.Getenv("GOVC_TUNNEL_PROXY_PORT")
	if port != "" {
		host += ":" + port
	}

	c.p = &url.URL{
		Scheme: "http",
		Host:   host,
	}
	t.Proxy = func(r *http.Request) (*url.URL, error) {
		// Only sdk requests should be proxied
		if r.URL.Path == "/sdk" {
			return c.p, nil
		}
		return http.ProxyFromEnvironment(r)
	}

	// Rewrite url Host to use the sdk tunnel, required for a certificate request.
	c.u.Host = sdkTunnel
}
示例#8
0
// dial dials the host specified by req, using TLS if appropriate, optionally
// using a proxy server if one is configured via environment variables.
func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) {
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, err
	}

	if proxyURL == nil {
		return s.dialWithoutProxy(req.URL)
	}

	// ensure we use a canonical host with proxyReq
	targetHost := netutil.CanonicalAddr(req.URL)

	// proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support
	proxyReq := http.Request{
		Method: "CONNECT",
		URL:    &url.URL{},
		Host:   targetHost,
	}

	proxyDialConn, err := s.dialWithoutProxy(proxyURL)
	if err != nil {
		return nil, err
	}

	proxyClientConn := httputil.NewProxyClientConn(proxyDialConn, nil)
	_, err = proxyClientConn.Do(&proxyReq)
	if err != nil && err != httputil.ErrPersistEOF {
		return nil, err
	}

	rwc, _ := proxyClientConn.Hijack()

	if req.URL.Scheme != "https" {
		return rwc, nil
	}

	host, _, err := net.SplitHostPort(req.URL.Host)
	if err != nil {
		return nil, err
	}

	if len(s.tlsConfig.ServerName) == 0 {
		s.tlsConfig.ServerName = host
	}

	tlsConn := tls.Client(rwc, s.tlsConfig)

	// need to manually call Handshake() so we can call VerifyHostname() below
	if err := tlsConn.Handshake(); err != nil {
		return nil, err
	}

	if err := tlsConn.VerifyHostname(host); err != nil {
		return nil, err
	}

	return tlsConn, nil
}
示例#9
0
func getProxy(req *http.Request) (*url.URL, error) {
	if config.ProxyServer != "" {
		u, err := url.Parse(config.ProxyServer)
		if err == nil {
			return u, nil
		}
	}
	return http.ProxyFromEnvironment(req)
}
示例#10
0
文件: sqs.go 项目: nalin-humin/goamz
func (s *SQS) query(queueUrl string, params map[string]string, resp interface{}) (err error) {
	var url_ *url.URL

	if queueUrl != "" && len(queueUrl) > len(s.Region.SQSEndpoint) {
		url_, err = url.Parse(queueUrl)
	} else {
		url_, err = url.Parse(s.Region.SQSEndpoint)
	}

	if err != nil {
		return err
	}

	params["Version"] = "2012-11-05"
	hreq, err := http.NewRequest("POST", url_.String(), strings.NewReader(multimap(params).Encode()))
	if err != nil {
		return err
	}

	// respect the environmnet's proxy settings
	if prox_url, _ := http.ProxyFromEnvironment(hreq); prox_url != nil {
		hreq.URL = prox_url
	}

	hreq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	hreq.Header.Set("X-Amz-Date", time.Now().UTC().Format(aws.ISO8601BasicFormat))

	if s.Auth.Token() != "" {
		hreq.Header.Set("X-Amz-Security-Token", s.Auth.Token())
	}

	signer := aws.NewV4Signer(s.Auth, "sqs", s.Region)
	signer.Sign(hreq)

	r, err := http.DefaultClient.Do(hreq)

	if err != nil {
		return err
	}

	defer r.Body.Close()

	if debug {
		dump, _ := httputil.DumpResponse(r, true)
		log.Printf("DUMP:\n", string(dump))
	}

	if r.StatusCode != 200 {
		return buildError(r)
	}
	err = xml.NewDecoder(r.Body).Decode(resp)
	io.Copy(ioutil.Discard, r.Body)

	return err
}
示例#11
0
文件: forge.go 项目: xorpaul/g10k
// getMetadataForgeModule queries the configured Puppet Forge and return
func getMetadataForgeModule(fm ForgeModule) ForgeModule {
	baseUrl := config.Forge.Baseurl
	if len(fm.baseUrl) > 0 {
		baseUrl = fm.baseUrl
	}
	url := baseUrl + "/v3/releases/" + fm.author + "-" + fm.name + "-" + fm.version
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
	req.Header.Set("Connection", "close")
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		Fatalf("getMetadataForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
	}
	client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
	before := time.Now()
	Debugf("GETing " + url)
	resp, err := client.Do(req)
	duration := time.Since(before).Seconds()
	Verbosef("GETing Forge metadata from " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
	mutex.Lock()
	syncForgeTime += duration
	mutex.Unlock()
	if err != nil {
		Fatalf("getMetadataForgeModule(): Error while querying metadata for Forge module " + fm.name + " from " + url + ": " + err.Error())
	}
	defer resp.Body.Close()

	if resp.Status == "200 OK" {
		body, err := ioutil.ReadAll(resp.Body)

		if err != nil {
			Fatalf("getMetadataForgeModule(): Error while reading response body for Forge module " + fm.name + " from " + url + ": " + err.Error())
		}

		before := time.Now()
		currentRelease := gjson.Parse(string(body)).Map()
		duration := time.Since(before).Seconds()
		modulemd5sum := currentRelease["file_md5"].String()
		moduleFilesize := currentRelease["file_size"].Int()
		Debugf("module: " + fm.author + "/" + fm.name + " modulemd5sum: " + modulemd5sum + " moduleFilesize: " + strconv.FormatInt(moduleFilesize, 10))

		mutex.Lock()
		forgeJsonParseTime += duration
		mutex.Unlock()

		return ForgeModule{md5sum: modulemd5sum, fileSize: moduleFilesize}
	} else {
		Fatalf("getMetadataForgeModule(): Unexpected response code while GETing " + url + " " + resp.Status)
	}
	return ForgeModule{}
}
示例#12
0
文件: main.go 项目: gowroc/meetups
func NewMultipleHostReverseProxy(reg Registry) *httputil.ReverseProxy {
	return &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "http"
			req.URL.Host = req.Host
		},
		Transport: &http.Transport{
			Proxy: func(req *http.Request) (*url.URL, error) {
				return http.ProxyFromEnvironment(req)
			},
			Dial: func(network, addr string) (net.Conn, error) {
				return loadBalance(network, addr, reg)
			},
			TLSHandshakeTimeout: 10 * time.Second,
		},
	}
}
示例#13
0
func (this *curler) new_proxy() proxy_func {
	if this.proxy_policy == CurlProxyPolicyAlwayseProxy {
		return http.ProxyFromEnvironment
	}
	if this.proxy_policy == CurlProxyPolicyNoProxy {
		return nil
	}
	if this.ruler != nil {
		return func(req *http.Request) (*url.URL, error) {
			uri := req.URL.String()
			blocked := this.ruler.IsBlocked(uri)
			if blocked {
				return http.ProxyFromEnvironment(req)
			}
			return nil, nil
		}
	}
	return nil
}
示例#14
0
// NewMultipleHostReverseProxy creates a reverse proxy that will randomly
// select a host from the passed `targets`
func newPeterHostReverseProxy(target *url.URL) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		fmt.Println("")
		req.URL.Scheme = target.Scheme
		req.URL.Host = target.Host
		req.URL.Path = target.Path
		fmt.Printf("DIRECTOR====> Scheme='%s' Host='%s' Path='%s'\n",
			req.URL.Scheme, req.URL.Host, req.URL.Path)
		os.Stdout.Sync()
	}

	return &httputil.ReverseProxy{
		Director: director,
		Transport: &http.Transport{
			Proxy: func(req *http.Request) (*url.URL, error) {
				fmt.Printf("PROXY   ====>req=%#v\n", req)
				length := req.ContentLength
				fmt.Printf("length=%d\n", length)
				bodyReader := req.Body
				body := readBody(bodyReader, int(length))
				fmt.Printf("body=%v\n", body)
				os.Stdout.Sync()
				return http.ProxyFromEnvironment(req)
			},
			Dial: func(network, addr string) (net.Conn, error) {
				fmt.Printf("DIAL   ====>addr='%s'\n", addr)
				conn, err := (&net.Dialer{
					Timeout:   30 * time.Second,
					KeepAlive: 30 * time.Second,
				}).Dial(network, addr)
				if err != nil {
					println("Error during DIAL:", err.Error())
				}
				os.Stdout.Sync()
				return conn, err
			},
			TLSHandshakeTimeout: 10 * time.Second,
		},
	}
}
示例#15
0
// Creates a new Twitter client with the supplied OAuth configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
//     export HTTP_PROXY=http://localhost:8888
//
// When using a proxy, disable TLS certificate verification with the following:
//     export TLS_INSECURE=1
func NewClient(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client {
	var (
		host      = "api.twitter.com"
		base      = "https://" + host
		req, _    = http.NewRequest("GET", "https://api.twitter.com", nil)
		proxy, _  = http.ProxyFromEnvironment(req)
		transport *http.Transport
		tlsconfig *tls.Config
	)
	if proxy != nil {
		tlsconfig = &tls.Config{
			InsecureSkipVerify: getEnvEitherCase("TLS_INSECURE") != "",
		}
		if tlsconfig.InsecureSkipVerify {
			log.Printf("WARNING: SSL cert verification  disabled\n")
		}
		transport = &http.Transport{
			Proxy:           http.ProxyURL(proxy),
			TLSClientConfig: tlsconfig,
		}
	} else {
		transport = &http.Transport{}
	}
	return &Client{
		Host: host,
		HttpClient: &http.Client{
			Transport: transport,
		},
		User:     user,
		AppToken: nil,
		OAuth: &oauth1a.Service{
			RequestURL:   base + "/oauth/request_token",
			AuthorizeURL: base + "/oauth/authorize",
			AccessURL:    base + "/oauth/access_token",
			ClientConfig: config,
			Signer:       new(oauth1a.HmacSha1Signer),
		},
	}
}
示例#16
0
// Creates a new Infoblox client with the supplied user/pass configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
//     export HTTP_PROXY=http://localhost:8888
//
// When using a proxy, disable TLS certificate verification with the following:
//    sslVerify = false
//
// To save and re-use infoblox session cookies, set useCookies = true
// NOTE: The infoblox cookie uses a comma separated string, and requires golang 1.3+ to be correctly stored.
//
func NewClient(host, username, password string, sslVerify, useCookies bool) *Client {
	var (
		req, _    = http.NewRequest("GET", host, nil)
		proxy, _  = http.ProxyFromEnvironment(req)
		transport *http.Transport
		tlsconfig *tls.Config
	)
	tlsconfig = &tls.Config{
		InsecureSkipVerify: !sslVerify,
	}
	if tlsconfig.InsecureSkipVerify {
		log.Printf("WARNING: SSL cert verification  disabled\n")
	}
	transport = &http.Transport{
		TLSClientConfig: tlsconfig,
	}
	if proxy != nil {
		transport.Proxy = http.ProxyURL(proxy)
	}

	client := &Client{
		Host: host,
		HttpClient: &http.Client{
			Transport: transport,
		},
		Username:   username,
		Password:   password,
		UseCookies: useCookies,
	}
	if useCookies {
		options := cookiejar.Options{
			PublicSuffixList: publicsuffix.List,
		}
		jar, _ := cookiejar.New(&options)
		client.HttpClient.Jar = jar
	}
	return client

}
示例#17
0
func dialViaHTTPProxy(server string, port uint16) (*tls.Conn, error) {
	xmppHost := fmt.Sprintf("%s:%d", server, port)
	fakeRequest := http.Request{
		URL: &url.URL{
			Scheme: "https",
			Host:   xmppHost,
		},
	}
	proxyURL, err := http.ProxyFromEnvironment(&fakeRequest)
	if err != nil {
		return nil, err
	}
	if proxyURL == nil {
		return nil, nil
	}

	dialer := net.Dialer{
		KeepAlive: netKeepAlive,
		Timeout:   netTimeout,
	}
	conn, err := dialer.Dial("tcp", proxyURL.Host)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to HTTP proxy server: %s", err)
	}

	fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", xmppHost, xmppHost)

	response, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"})
	if err != nil {
		return nil, err
	}
	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Failed to connect to proxy: %s", response.Status)
	}

	return addTLS(server, conn)
}
示例#18
0
文件: bat.go 项目: cofyc/bat
func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) > 0 {
		args = filter(args)
	}
	if ver {
		fmt.Println("Version:", version)
		os.Exit(2)
	}
	if printV != "A" && printV != "B" {
		defaultSetting.DumpBody = false
	}
	var stdin []byte
	if runtime.GOOS != "windows" {
		fi, err := os.Stdin.Stat()
		if err != nil {
			panic(err)
		}
		if fi.Size() != 0 {
			stdin, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				log.Fatal("Read from Stdin", err)
			}
		}
	}

	if *URL == "" {
		usage()
	}
	if strings.HasPrefix(*URL, ":") {
		urlb := []byte(*URL)
		if *URL == ":" {
			*URL = "http://localhost/"
		} else if len(*URL) > 1 && urlb[1] != '/' {
			*URL = "http://localhost" + *URL
		} else {
			*URL = "http://localhost" + string(urlb[1:])
		}
	}
	if !strings.HasPrefix(*URL, "http://") && !strings.HasPrefix(*URL, "https://") {
		*URL = "http://" + *URL
	}
	u, err := url.Parse(*URL)
	if err != nil {
		log.Fatal(err)
	}
	if auth != "" {
		userpass := strings.Split(auth, ":")
		if len(userpass) == 2 {
			u.User = url.UserPassword(userpass[0], userpass[1])
		} else {
			u.User = url.User(auth)
		}
	}
	*URL = u.String()
	httpreq := getHTTP(*method, *URL, args)
	if u.User != nil {
		password, _ := u.User.Password()
		httpreq.GetRequest().SetBasicAuth(u.User.Username(), password)
	}
	// Insecure SSL Support
	if insecureSSL {
		httpreq.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
	}
	// Proxy Support
	if proxy != "" {
		purl, err := url.Parse(proxy)
		if err != nil {
			log.Fatal("Proxy Url parse err", err)
		}
		httpreq.SetProxy(http.ProxyURL(purl))
	} else {
		eurl, err := http.ProxyFromEnvironment(httpreq.GetRequest())
		if err != nil {
			log.Fatal("Environment Proxy Url parse err", err)
		}
		httpreq.SetProxy(http.ProxyURL(eurl))
	}
	if body != "" {
		httpreq.Body(body)
	}
	if len(stdin) > 0 {
		var j interface{}
		d := json.NewDecoder(bytes.NewReader(stdin))
		d.UseNumber()
		err = d.Decode(&j)
		if err != nil {
			httpreq.Body(stdin)
		} else {
			httpreq.JsonBody(j)
		}
	}

	// AB bench
	if bench {
		httpreq.Debug(false)
		RunBench(httpreq)
		return
	}
	res, err := httpreq.Response()
	if err != nil {
		log.Fatalln("can't get the url", err)
	}

	// download file
	if download {
		var fl string
		if disposition := res.Header.Get("Content-Disposition"); disposition != "" {
			fls := strings.Split(disposition, ";")
			for _, f := range fls {
				f = strings.TrimSpace(f)
				if strings.HasPrefix(f, "filename=") {
					fl = strings.TrimLeft(f, "filename=")
				}
			}
		}
		if fl == "" {
			_, fl = filepath.Split(u.Path)
		}
		fd, err := os.OpenFile(fl, os.O_RDWR|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("can't create file", err)
		}
		if runtime.GOOS != "windows" {
			fmt.Println(Color(res.Proto, Magenta), Color(res.Status, Green))
			for k, v := range res.Header {
				fmt.Println(Color(k, Gray), ":", Color(strings.Join(v, " "), Cyan))
			}
		} else {
			fmt.Println(res.Proto, res.Status)
			for k, v := range res.Header {
				fmt.Println(k, ":", strings.Join(v, " "))
			}
		}
		fmt.Println("")
		contentLength := res.Header.Get("Content-Length")
		var total int64
		if contentLength != "" {
			total, _ = strconv.ParseInt(contentLength, 10, 64)
		}
		fmt.Printf("Downloading to \"%s\"\n", fl)
		pb := NewProgressBar(total)
		pb.Start()
		multiWriter := io.MultiWriter(fd, pb)
		_, err = io.Copy(multiWriter, res.Body)
		if err != nil {
			log.Fatal("Can't Write the body into file", err)
		}
		pb.Finish()
		defer fd.Close()
		defer res.Body.Close()
		return
	}

	if runtime.GOOS != "windows" {
		fi, err := os.Stdout.Stat()
		if err != nil {
			panic(err)
		}
		if fi.Mode()&os.ModeDevice == os.ModeDevice {
			if printV == "A" || printV == "H" || printV == "B" {
				dump := httpreq.DumpRequest()
				if printV == "B" {
					dps := strings.Split(string(dump), "\n")
					for i, line := range dps {
						if len(strings.Trim(line, "\r\n ")) == 0 {
							dump = []byte(strings.Join(dps[i:], "\n"))
							break
						}
					}
				}
				fmt.Println(ColorfulRequest(string(dump)))
				fmt.Println("")
			}
			if printV == "A" || printV == "h" {
				fmt.Println(Color(res.Proto, Magenta), Color(res.Status, Green))
				for k, v := range res.Header {
					fmt.Println(Color(k, Gray), ":", Color(strings.Join(v, " "), Cyan))
				}
				fmt.Println("")
			}
			if printV == "A" || printV == "b" {
				body := formatResponseBody(res, httpreq, pretty)
				fmt.Println(ColorfulResponse(body, res.Header.Get("Content-Type")))
			}
		} else {
			body := formatResponseBody(res, httpreq, pretty)
			_, err = os.Stdout.WriteString(body)
			if err != nil {
				log.Fatal(err)
			}
		}
	} else {
		if printV == "A" || printV == "H" || printV == "B" {
			dump := httpreq.DumpRequest()
			if printV == "B" {
				dps := strings.Split(string(dump), "\n")
				for i, line := range dps {
					if len(strings.Trim(line, "\r\n ")) == 0 {
						dump = []byte(strings.Join(dps[i:], "\n"))
						break
					}
				}
			}
			fmt.Println(string(dump))
			fmt.Println("")
		}
		if printV == "A" || printV == "h" {
			fmt.Println(res.Proto, res.Status)
			for k, v := range res.Header {
				fmt.Println(k, ":", strings.Join(v, " "))
			}
			fmt.Println("")
		}
		if printV == "A" || printV == "b" {
			body := formatResponseBody(res, httpreq, pretty)
			fmt.Println(body)
		}
	}
}
示例#19
0
func SmartProxy(req *http.Request) (*url.URL, error) {
	return http.ProxyFromEnvironment(req)
}
示例#20
0
文件: forge.go 项目: xorpaul/g10k
func downloadForgeModule(name string, version string, fm ForgeModule, retryCount int) {
	funcName := funcName()
	var wgForgeModule sync.WaitGroup

	extractR, extractW := io.Pipe()
	saveFileR, saveFileW := io.Pipe()

	//url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz"
	fileName := name + "-" + version + ".tar.gz"

	if !fileExists(config.ForgeCacheDir + name + "-" + version) {
		baseUrl := config.Forge.Baseurl
		if len(fm.baseUrl) > 0 {
			baseUrl = fm.baseUrl
		}
		url := baseUrl + "/v3/files/" + fileName
		req, err := http.NewRequest("GET", url, nil)
		req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
		req.Header.Set("Connection", "close")
		proxyURL, err := http.ProxyFromEnvironment(req)
		if err != nil {
			Fatalf(funcName + "(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
		}
		client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
		before := time.Now()
		Debugf("GETing " + url)
		resp, err := client.Do(req)
		duration := time.Since(before).Seconds()
		Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
		mutex.Lock()
		syncForgeTime += duration
		mutex.Unlock()
		if err != nil {
			Fatalf(funcName + "(): Error while GETing Forge module " + name + " from " + url + ": " + err.Error())
		}
		defer resp.Body.Close()

		if resp.Status == "200 OK" {
			wgForgeModule.Add(1)
			go func() {
				defer wgForgeModule.Done()
				Debugf(funcName + "(): Trying to create " + config.ForgeCacheDir + fileName)
				out, err := os.Create(config.ForgeCacheDir + fileName)
				if err != nil {
					Fatalf(funcName + "(): Error while creating file for Forge module " + config.ForgeCacheDir + fileName + " Error: " + err.Error())
				}
				defer out.Close()
				io.Copy(out, saveFileR)
				Debugf(funcName + "(): Finished creating " + config.ForgeCacheDir + fileName)
			}()
			wgForgeModule.Add(1)
			go extractForgeModule(&wgForgeModule, extractR, fileName)
			wgForgeModule.Add(1)
			go func() {
				defer wgForgeModule.Done()

				// after completing the copy, we need to close
				// the PipeWriters to propagate the EOF to all
				// PipeReaders to avoid deadlock
				defer extractW.Close()
				defer saveFileW.Close()

				var mw io.Writer
				// build the multiwriter for all the pipes
				mw = io.MultiWriter(extractW, saveFileW)

				// copy the data into the multiwriter
				if _, err := io.Copy(mw, resp.Body); err != nil {
					Fatalf("Error while writing to MultiWriter " + err.Error())
				}
			}()
		} else {
			Fatalf(funcName + "(): Unexpected response code while GETing " + url + " " + resp.Status)
		}
	} else {
		Debugf("Using cache for Forge module " + name + " version: " + version)
	}
	wgForgeModule.Wait()

	if checkSum || fm.sha256sum != "" {
		fm.version = version
		if doForgeModuleIntegrityCheck(fm) {
			if retryCount == 0 {
				Fatalf("downloadForgeModule(): giving up for Puppet module " + name + " version: " + version)
			}
			Warnf("Retrying...")
			purgeDir(config.ForgeCacheDir+fileName, "downloadForgeModule()")
			purgeDir(strings.Replace(config.ForgeCacheDir+fileName, ".tar.gz", "/", -1), "downloadForgeModule()")
			// retry if hash sum mismatch found
			downloadForgeModule(name, version, fm, retryCount-1)
		}
	}

}
示例#21
0
文件: context.go 项目: igofman/mig
func initMQ(orig_ctx Context, try_proxy bool, proxy string) (ctx Context, err error) {
	ctx = orig_ctx
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("initMQ() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{Desc: "leaving initMQ()"}.Debug()
	}()

	//Define the AMQP binding
	ctx.MQ.Bind.Queue = fmt.Sprintf("mig.agt.%s", ctx.Agent.QueueLoc)
	ctx.MQ.Bind.Key = fmt.Sprintf("mig.agt.%s", ctx.Agent.QueueLoc)

	// parse the dial string and use TLS if using amqps
	amqp_uri, err := amqp.ParseURI(AMQPBROKER)
	if err != nil {
		panic(err)
	}
	ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("AMQP: host=%s, port=%d, vhost=%s", amqp_uri.Host, amqp_uri.Port, amqp_uri.Vhost)}.Debug()
	if amqp_uri.Scheme == "amqps" {
		ctx.MQ.UseTLS = true
	}

	// create an AMQP configuration with specific timers
	var dialConfig amqp.Config
	dialConfig.Heartbeat = 2 * ctx.Sleeper
	if try_proxy {
		// if in try_proxy mode, the agent will try to connect to the relay using a CONNECT proxy
		// but because CONNECT is a HTTP method, not available in AMQP, we need to establish
		// that connection ourselves, and give it back to the amqp.DialConfig method
		if proxy == "" {
			// try to get the proxy from the environemnt (variable HTTP_PROXY)
			target := "http://" + amqp_uri.Host + ":" + fmt.Sprintf("%d", amqp_uri.Port)
			req, err := http.NewRequest("GET", target, nil)
			if err != nil {
				panic(err)
			}
			proxy_url, err := http.ProxyFromEnvironment(req)
			if err != nil {
				panic(err)
			}
			if proxy_url == nil {
				panic("Failed to find a suitable proxy in environment")
			}
			proxy = proxy_url.Host
			ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Found proxy at %s", proxy)}.Debug()
		}
		ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Connecting via proxy %s", proxy)}.Debug()
		dialConfig.Dial = func(network, addr string) (conn net.Conn, err error) {
			// connect to the proxy
			conn, err = net.DialTimeout("tcp", proxy, 5*time.Second)
			if err != nil {
				return
			}
			// write a CONNECT request in the tcp connection
			fmt.Fprintf(conn, "CONNECT "+addr+" HTTP/1.1\r\nHost: "+addr+"\r\n\r\n")
			// verify status is 200, and flush the buffer
			status, err := bufio.NewReader(conn).ReadString('\n')
			if err != nil {
				return
			}
			if status == "" || len(status) < 12 {
				err = fmt.Errorf("Invalid status received from proxy: '%s'", status[0:len(status)-2])
				return
			}
			// 9th character in response should be "2"
			// HTTP/1.0 200 Connection established
			//          ^
			if status[9] != '2' {
				err = fmt.Errorf("Invalid status received from proxy: '%s'", status[0:len(status)-2])
				return
			}
			ctx.Agent.Env.IsProxied = true
			ctx.Agent.Env.Proxy = proxy
			return
		}
	} else {
		dialConfig.Dial = func(network, addr string) (net.Conn, error) {
			return net.DialTimeout(network, addr, 5*time.Second)
		}
	}

	if ctx.MQ.UseTLS {
		ctx.Channels.Log <- mig.Log{Desc: "Loading AMQPS TLS parameters"}.Debug()
		// import the client certificates
		cert, err := tls.X509KeyPair(AGENTCERT, AGENTKEY)
		if err != nil {
			panic(err)
		}

		// import the ca cert
		ca := x509.NewCertPool()
		if ok := ca.AppendCertsFromPEM(CACERT); !ok {
			panic("failed to import CA Certificate")
		}
		TLSconfig := tls.Config{Certificates: []tls.Certificate{cert},
			RootCAs:            ca,
			InsecureSkipVerify: false,
			Rand:               rand.Reader}

		dialConfig.TLSClientConfig = &TLSconfig
	}
	// Open AMQP connection
	ctx.Channels.Log <- mig.Log{Desc: "Establishing connection to relay"}.Debug()
	ctx.MQ.conn, err = amqp.DialConfig(AMQPBROKER, dialConfig)
	if err != nil {
		ctx.Channels.Log <- mig.Log{Desc: "Connection failed"}.Debug()
		panic(err)
	}

	ctx.MQ.Chan, err = ctx.MQ.conn.Channel()
	if err != nil {
		panic(err)
	}

	// Limit the number of message the channel will receive at once
	err = ctx.MQ.Chan.Qos(1, // prefetch count (in # of msg)
		0,     // prefetch size (in bytes)
		false) // is global

	_, err = ctx.MQ.Chan.QueueDeclare(ctx.MQ.Bind.Queue, // Queue name
		true,  // is durable
		false, // is autoDelete
		false, // is exclusive
		false, // is noWait
		nil)   // AMQP args
	if err != nil {
		panic(err)
	}

	err = ctx.MQ.Chan.QueueBind(ctx.MQ.Bind.Queue, // Queue name
		ctx.MQ.Bind.Key,    // Routing key name
		mig.Mq_Ex_ToAgents, // Exchange name
		false,              // is noWait
		nil)                // AMQP args
	if err != nil {
		panic(err)
	}

	// Consume AMQP message into channel
	ctx.MQ.Bind.Chan, err = ctx.MQ.Chan.Consume(ctx.MQ.Bind.Queue, // queue name
		"",    // some tag
		false, // is autoAck
		false, // is exclusive
		false, // is noLocal
		false, // is noWait
		nil)   // AMQP args
	if err != nil {
		panic(err)
	}

	return
}
示例#22
0
文件: forge.go 项目: mikey179/g10k
func queryForgeApi(name string, file string) ForgeResult {
	//url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1)
	url := "https://forgeapi.puppetlabs.com:443/v3/modules?query=" + name
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal("queryForgeApi(): Error creating GET request for Puppetlabs forge API", err)
		os.Exit(1)
	}
	if fileInfo, err := os.Stat(file); err == nil {
		Debugf("queryForgeApi(): adding If-Modified-Since:" + string(fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + " to Forge query")
		req.Header.Set("If-Modified-Since", fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
	}
	req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
	req.Header.Set("Connection", "close")

	proxyUrl, err := http.ProxyFromEnvironment(req)
	if err != nil {
		log.Fatal("queryForgeApi(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err)
		os.Exit(1)
	}
	client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
	before := time.Now()
	resp, err := client.Do(req)
	duration := time.Since(before).Seconds()
	Verbosef("Querying Forge API " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
	mutex.Lock()
	syncForgeTime += duration
	mutex.Unlock()
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	if resp.Status == "200 OK" {
		// need to get latest version
		body, err := ioutil.ReadAll(resp.Body)

		//fmt.Println(string(body))
		reCurrent := regexp.MustCompile("\\s*\"current_release\": {\n\\s*\"uri\": \"([^\"]+)\",")
		if m := reCurrent.FindStringSubmatch(string(body)); len(m) > 1 {
			//fmt.Println(m[1])
			if strings.Count(m[1], "-") < 2 {
				log.Fatal("queryForgeApi(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", m[1], " should contain three '-' characters")
				os.Exit(1)
			} else {
				version := strings.Split(m[1], "-")[2]
				Debugf("queryForgeApi(): found version " + version + " for " + name + "-latest")
				mutex.Lock()
				latestForgeModules[name] = version
				mutex.Unlock()
				return ForgeResult{true, version}
			}
		}

		if err != nil {
			panic(err)
		}
		return ForgeResult{false, ""}
	} else if resp.Status == "304 Not Modified" {
		Debugf("queryForgeApi(): Got 304 nothing to do for module " + name)
		return ForgeResult{false, ""}
	} else {
		Debugf("queryForgeApi(): Unexpected response code " + resp.Status)
		return ForgeResult{false, ""}
	}
}
示例#23
0
// RestAPICall - general rest method caller
func (c *Client) RestAPICall(method Method, path string, options interface{}) ([]byte, error) {
	log.Debugf("RestAPICall %s - %s%s", method, utils.Sanatize(c.Endpoint), path)

	var (
		Url *url.URL
		err error
		req *http.Request
	)

	Url, err = url.Parse(utils.Sanatize(c.Endpoint))
	if err != nil {
		return nil, err
	}
	Url.Path += path

	// Manage the query string
	c.GetQueryString(Url)

	log.Debugf("*** url => %s", Url.String())
	log.Debugf("*** method => %s", method.String())

	// parse url
	reqUrl, err := url.Parse(Url.String())
	if err != nil {
		return nil, fmt.Errorf("Error with request: %v - %q", Url, err)
	}

	// handle options
	if options != nil {
		OptionsJSON, err := json.Marshal(options)
		if err != nil {
			return nil, err
		}
		log.Debugf("*** options => %+v", bytes.NewBuffer(OptionsJSON))
		req, err = http.NewRequest(method.String(), reqUrl.String(), bytes.NewBuffer(OptionsJSON))
	} else {
		req, err = http.NewRequest(method.String(), reqUrl.String(), nil)
	}

	if err != nil {
		return nil, fmt.Errorf("Error with request: %v - %q", Url, err)
	}

	// setup proxy
	proxyUrl, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, fmt.Errorf("Error with proxy: %v - %q", proxyUrl, err)
	}
	if proxyUrl != nil {
		tr.Proxy = http.ProxyURL(proxyUrl)
		log.Debugf("*** proxy => %+v", tr.Proxy)
	}

	// build the auth headerU
	for k, v := range c.Option.Headers {
		log.Debugf("Headers -> %s -> %+v\n", k, v)
		req.Header.Add(k, v)
	}

	// req.SetBasicAuth(c.User, c.APIKey)
	req.Method = fmt.Sprintf("%s", method.String())

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// TODO: CLeanup Later
	// DEBUGGING WHILE WE WORK
	// DEBUGGING WHILE WE WORK
	// fmt.Printf("METHOD --> %+v\n",method)
	log.Debugf("REQ    --> %+v\n", req)
	log.Debugf("RESP   --> %+v\n", resp)
	log.Debugf("ERROR  --> %+v\n", err)
	// DEBUGGING WHILE WE WORK

	data, err := ioutil.ReadAll(resp.Body)

	if !c.isOkStatus(resp.StatusCode) {
		type apiErr struct {
			Err string `json:"details"`
		}
		var outErr apiErr
		json.Unmarshal(data, &outErr)
		return nil, fmt.Errorf("Error in response: %s\n Response Status: %s", outErr.Err, resp.Status)
	}

	if err != nil {
		return nil, err
	}

	return data, nil
}
示例#24
0
文件: forge.go 项目: mikey179/g10k
func downloadForgeModule(name string, version string) {
	//url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz"
	fileName := name + "-" + version + ".tar.gz"
	if _, err := os.Stat(config.ForgeCacheDir + name + "-" + version); os.IsNotExist(err) {
		url := "https://forgeapi.puppetlabs.com/v3/files/" + fileName
		req, err := http.NewRequest("GET", url, nil)
		req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
		req.Header.Set("Connection", "close")
		proxyUrl, err := http.ProxyFromEnvironment(req)
		if err != nil {
			log.Fatal("downloadForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err)
			os.Exit(1)
		}
		client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
		before := time.Now()
		resp, err := client.Do(req)
		duration := time.Since(before).Seconds()
		Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
		mutex.Lock()
		syncForgeTime += duration
		mutex.Unlock()
		if err != nil {
			log.Print("downloadForgeModule(): Error while GETing Forge module ", name, " from ", url, ": ", err)
			os.Exit(1)
		}
		defer resp.Body.Close()

		if resp.Status == "200 OK" {
			Debugf("downloadForgeModule(): Trying to create " + config.ForgeCacheDir + fileName)
			out, err := os.Create(config.ForgeCacheDir + fileName)
			if err != nil {
				log.Print("downloadForgeModule(): Error while creating file for Forge module "+config.ForgeCacheDir+fileName, err)
				os.Exit(1)
			}
			defer out.Close()
			io.Copy(out, resp.Body)
			file, err := os.Open(config.ForgeCacheDir + fileName)

			if err != nil {
				fmt.Println("downloadForgeModule(): Error while opening file", file, err)
				os.Exit(1)
			}

			defer file.Close()

			var fileReader io.ReadCloser = resp.Body
			if strings.HasSuffix(fileName, ".gz") {
				if fileReader, err = pgzip.NewReader(file); err != nil {

					fmt.Println("downloadForgeModule(): pgzip reader error for module ", fileName, " error:", err)
					os.Exit(1)
				}
				defer fileReader.Close()
			}

			tarBallReader := tar.NewReader(fileReader)
			if err = os.Chdir(config.ForgeCacheDir); err != nil {

				fmt.Println("downloadForgeModule(): error while chdir to", config.ForgeCacheDir, err)
				os.Exit(1)
			}
			for {
				header, err := tarBallReader.Next()
				if err != nil {
					if err == io.EOF {
						break
					}
					fmt.Println("downloadForgeModule(): error while tar reader.Next() for ", fileName, err)
					os.Exit(1)
				}

				// get the individual filename and extract to the current directory
				filename := header.Name
				//Debugf("downloadForgeModule(): Trying to extract file" + filename)

				switch header.Typeflag {
				case tar.TypeDir:
					// handle directory
					//fmt.Println("Creating directory :", filename)
					//err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755 if you prefer
					err = os.MkdirAll(filename, os.FileMode(0755)) // or use 0755 if you prefer

					if err != nil {
						fmt.Println("downloadForgeModule(): error while MkdirAll()", filename, err)
						os.Exit(1)
					}

				case tar.TypeReg:
					// handle normal file
					//fmt.Println("Untarring :", filename)
					writer, err := os.Create(filename)

					if err != nil {
						fmt.Println("downloadForgeModule(): error while Create()", filename, err)
						os.Exit(1)
					}

					io.Copy(writer, tarBallReader)

					err = os.Chmod(filename, os.FileMode(0644))

					if err != nil {
						fmt.Println("downloadForgeModule(): error while Chmod()", filename, err)
						os.Exit(1)
					}

					writer.Close()
				default:
					fmt.Printf("Unable to untar type : %c in file %s", header.Typeflag, filename)
				}
			}

		} else {
			log.Print("downloadForgeModule(): Unexpected response code while GETing " + url + resp.Status)
			os.Exit(1)
		}
	} else {
		Debugf("downloadForgeModule(): Using cache for Forge module " + name + " version: " + version)
	}
}
示例#25
0
// websocketConn establishes a connection to the given URL, respecting any proxy configuration in the environment.
// A standard proxying setup involves setting the following environment variables
// (may be listed in /etc/ecs/ecs.config if using ecs-init):
// HTTP_PROXY=http://<your-proxy>/ # HTTPS_PROXY may be set instead or additionally
// NO_PROXY=169.254.169.254,/var/run/docker.sock # Directly connect to metadata service and docker socket
func (cs *ClientServerImpl) websocketConn(parsedURL *url.URL, request *http.Request) (net.Conn, error) {
	proxyURL, err := http.ProxyFromEnvironment(request)
	if err != nil {
		return nil, err
	}

	// url.Host might not have the port, but tls.Dial needs it
	targetHost := parsedURL.Host
	if !strings.Contains(targetHost, ":") {
		targetHost += ":443"
	}
	targetHostname, _, err := net.SplitHostPort(targetHost)
	if err != nil {
		return nil, err
	}

	tlsConfig := tls.Config{ServerName: targetHostname, InsecureSkipVerify: cs.AcceptInvalidCert}
	timeoutDialer := &net.Dialer{Timeout: wsConnectTimeout}

	if proxyURL == nil {
		// directly connect
		log.Info("Creating poll dialer", "host", parsedURL.Host)
		return tls.DialWithDialer(timeoutDialer, "tcp", targetHost, &tlsConfig)
	}

	// connect via proxy
	log.Info("Creating poll dialer", "proxy", proxyURL.Host, "host", parsedURL.Host)
	plainConn, err := timeoutDialer.Dial("tcp", proxyURL.Host)
	if err != nil {
		return nil, err
	}

	// TLS over an HTTP proxy via CONNECT taken from: https://golang.org/src/net/http/transport.go
	connectReq := &http.Request{
		Method: "CONNECT",
		URL:    &url.URL{Opaque: targetHost},
		Host:   targetHost,
		Header: make(http.Header),
	}

	if proxyUser := proxyURL.User; proxyUser != nil {
		username := proxyUser.Username()
		password, _ := proxyUser.Password()
		auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
		connectReq.Header.Set("Proxy-Authorization", "Basic "+auth)
	}

	connectReq.Write(plainConn)

	// Read response.
	// Okay to use and discard buffered reader here, because
	// TLS server will not speak until spoken to.
	br := bufio.NewReader(plainConn)
	resp, err := http.ReadResponse(br, connectReq)
	if err != nil {
		plainConn.Close()
		return nil, err
	}
	if resp.StatusCode != 200 {
		plainConn.Close()
		return nil, errors.New(resp.Status)
	}

	tlsConn := tls.Client(plainConn, &tlsConfig)

	return tlsConn, nil
}
示例#26
0
文件: forge.go 项目: xorpaul/g10k
func queryForgeAPI(name string, file string, fm ForgeModule) ForgeResult {
	//url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1)
	baseUrl := config.Forge.Baseurl
	if len(fm.baseUrl) > 0 {
		baseUrl = fm.baseUrl
	}
	//url := baseUrl + "/v3/modules?query=" + name
	url := baseUrl + "/v3/releases?module=" + name + "&owner=" + fm.author + "&sort_by=release_date&limit=1"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		Fatalf("queryForgeAPI(): Error creating GET request for Puppetlabs forge API" + err.Error())
	}
	if fileInfo, err := os.Stat(file); err == nil {
		Debugf("adding If-Modified-Since:" + string(fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + " to Forge query")
		req.Header.Set("If-Modified-Since", fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
	}
	req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
	req.Header.Set("Connection", "close")

	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		Fatalf("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
	}
	client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
	before := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		Fatalf("queryForgeAPI(): Error while issuing the HTTP request to " + url + " Error: " + err.Error())
	}
	duration := time.Since(before).Seconds()
	Verbosef("Querying Forge API " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")

	mutex.Lock()
	syncForgeTime += duration
	mutex.Unlock()
	defer resp.Body.Close()

	if resp.Status == "200 OK" {
		// need to get latest version
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			Fatalf("queryForgeAPI(): Error while reading response body for Forge module " + fm.name + " from " + url + ": " + err.Error())
		}

		before := time.Now()
		currentRelease := gjson.Get(string(body), "results.0").Map()

		duration := time.Since(before).Seconds()
		version := currentRelease["version"].String()
		modulemd5sum := currentRelease["file_md5"].String()
		moduleFilesize := currentRelease["file_size"].Int()

		mutex.Lock()
		forgeJsonParseTime += duration
		mutex.Unlock()

		Debugf("found version " + version + " for " + name + "-latest")
		latestForgeModules.Lock()
		latestForgeModules.m[name] = version
		latestForgeModules.Unlock()

		lastCheckedFile := config.ForgeCacheDir + name + "-latest-last-checked"
		Debugf("writing last-checked file " + lastCheckedFile)
		f, _ := os.Create(lastCheckedFile)
		defer f.Close()

		return ForgeResult{true, version, modulemd5sum, moduleFilesize}

	} else if resp.Status == "304 Not Modified" {
		Debugf("Got 304 nothing to do for module " + name)
		return ForgeResult{false, "", "", 0}
	} else {
		Debugf("Unexpected response code " + resp.Status)
		return ForgeResult{false, "", "", 0}
	}
}