// proxySettings will default to the default proxy settings if none are provided
// if settings are provided – they will override the environment variables
func (ro RequestOptions) proxySettings(req *http.Request) (*url.URL, error) {
// No proxies – lets use the default
if len(ro.Proxies) == 0 {
return http.ProxyFromEnvironment(req)
}
// There was a proxy specified – do we support the protocol?
if _, ok := ro.Proxies[req.URL.Scheme]; ok {
return ro.Proxies[req.URL.Scheme], nil
}
// Proxies were specified but not for any protocol that we use
return http.ProxyFromEnvironment(req)
}
// Creates a new Twitter client with the supplied OAuth configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
// export HTTP_PROXY=http://localhost:8888
func NewClient(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client {
var (
host = "api.twitter.com"
base = "https://" + host
req, _ = http.NewRequest("GET", "https://api.twitter.com", nil)
proxy, _ = http.ProxyFromEnvironment(req)
transport *http.Transport
)
if proxy != nil {
transport = &http.Transport{
Proxy: http.ProxyURL(proxy),
}
} else {
transport = &http.Transport{}
}
return &Client{
Host: host,
HttpClient: &http.Client{
Transport: transport,
},
User: user,
AppToken: nil,
OAuth: &oauth1a.Service{
RequestURL: base + "/oauth/request_token",
AuthorizeURL: base + "/oauth/authorize",
AccessURL: base + "/oauth/access_token",
ClientConfig: config,
Signer: new(oauth1a.HmacSha1Signer),
},
}
}
// NewClient creates a new Client object.
// The token can be empty if you plan on creating
// the token using the `CreateToken` func. If you want
// to use your existing token, you need to pass it as the paramter.
// It returns a Client object that is pre-configured for usage.
func NewClient(token string) *Client {
// Perhaps we should allow people to configure this through some
// variable. It would make the package almost 100% backwards compatible
// with the public api (https://developer.kwikdesk.com)
var (
host = "platform.kwikdesk.com"
base = "https://" + host
req, _ = http.NewRequest("GET", base, nil)
proxy, _ = http.ProxyFromEnvironment(req)
transport *http.Transport
)
transport = &http.Transport{}
if proxy != nil {
transport = &http.Transport{
Proxy: http.ProxyURL(proxy),
}
}
return &Client{
Host: host,
FullHost: base,
HttpClient: &http.Client{
Transport: transport,
},
ContentType: kdHTTPContentType,
XToken: token,
}
}
// Creates a new Infoblox client with the supplied user/pass configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
// export HTTP_PROXY=http://localhost:8888
//
// When using a proxy, disable TLS certificate verification with the following:
// sslVerify = false
func NewClient(host, username, password string, sslVerify bool) *Client {
var (
req, _ = http.NewRequest("GET", host, nil)
proxy, _ = http.ProxyFromEnvironment(req)
transport *http.Transport
tlsconfig *tls.Config
)
tlsconfig = &tls.Config{
InsecureSkipVerify: !sslVerify,
}
if tlsconfig.InsecureSkipVerify {
fmt.Println("WARNING: SSL cert verification disabled!")
}
transport = &http.Transport{
TLSClientConfig: tlsconfig,
}
if proxy != nil {
transport.Proxy = http.ProxyURL(proxy)
}
return &Client{
Host: host,
HttpClient: &http.Client{
Transport: transport,
},
Username: username,
Password: password,
}
}
func getProxy() *url.URL {
req, err := http.NewRequest("GET", "https://api.heroku.com", nil)
PrintError(err, false)
proxy, err := http.ProxyFromEnvironment(req)
PrintError(err, false)
return proxy
}
// NewMultipleHostReverseProxy creates a reverse proxy that will randomly
// select a host from the passed `targets`
func NewMultipleHostReverseProxy(targets []*url.URL) *httputil.ReverseProxy {
director := func(req *http.Request) {
println("CALLING DIRECTOR")
target := targets[rand.Int()%len(targets)]
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = target.Path
}
return &httputil.ReverseProxy{
Director: director,
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
println("CALLING PROXY")
return http.ProxyFromEnvironment(req)
},
Dial: func(network, addr string) (net.Conn, error) {
println("CALLING DIAL")
conn, err := (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial(network, addr)
if err != nil {
println("Error during DIAL:", err.Error())
}
return conn, err
},
TLSHandshakeTimeout: 10 * time.Second,
},
}
}
func (c *Client) SetCertificate(cert tls.Certificate) {
t := c.Client.Transport.(*http.Transport)
// Extension certificate
t.TLSClientConfig.Certificates = []tls.Certificate{cert}
// Proxy to vCenter host on port 80
host, _ := splitHostPort(c.u.Host)
// Should be no reason to change the default port other than testing
port := os.Getenv("GOVC_TUNNEL_PROXY_PORT")
if port != "" {
host += ":" + port
}
c.p = &url.URL{
Scheme: "http",
Host: host,
}
t.Proxy = func(r *http.Request) (*url.URL, error) {
// Only sdk requests should be proxied
if r.URL.Path == "/sdk" {
return c.p, nil
}
return http.ProxyFromEnvironment(r)
}
// Rewrite url Host to use the sdk tunnel, required for a certificate request.
c.u.Host = sdkTunnel
}
// dial dials the host specified by req, using TLS if appropriate, optionally
// using a proxy server if one is configured via environment variables.
func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) {
proxyURL, err := http.ProxyFromEnvironment(req)
if err != nil {
return nil, err
}
if proxyURL == nil {
return s.dialWithoutProxy(req.URL)
}
// ensure we use a canonical host with proxyReq
targetHost := netutil.CanonicalAddr(req.URL)
// proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support
proxyReq := http.Request{
Method: "CONNECT",
URL: &url.URL{},
Host: targetHost,
}
proxyDialConn, err := s.dialWithoutProxy(proxyURL)
if err != nil {
return nil, err
}
proxyClientConn := httputil.NewProxyClientConn(proxyDialConn, nil)
_, err = proxyClientConn.Do(&proxyReq)
if err != nil && err != httputil.ErrPersistEOF {
return nil, err
}
rwc, _ := proxyClientConn.Hijack()
if req.URL.Scheme != "https" {
return rwc, nil
}
host, _, err := net.SplitHostPort(req.URL.Host)
if err != nil {
return nil, err
}
if len(s.tlsConfig.ServerName) == 0 {
s.tlsConfig.ServerName = host
}
tlsConn := tls.Client(rwc, s.tlsConfig)
// need to manually call Handshake() so we can call VerifyHostname() below
if err := tlsConn.Handshake(); err != nil {
return nil, err
}
if err := tlsConn.VerifyHostname(host); err != nil {
return nil, err
}
return tlsConn, nil
}
func getProxy(req *http.Request) (*url.URL, error) {
if config.ProxyServer != "" {
u, err := url.Parse(config.ProxyServer)
if err == nil {
return u, nil
}
}
return http.ProxyFromEnvironment(req)
}
func (s *SQS) query(queueUrl string, params map[string]string, resp interface{}) (err error) {
var url_ *url.URL
if queueUrl != "" && len(queueUrl) > len(s.Region.SQSEndpoint) {
url_, err = url.Parse(queueUrl)
} else {
url_, err = url.Parse(s.Region.SQSEndpoint)
}
if err != nil {
return err
}
params["Version"] = "2012-11-05"
hreq, err := http.NewRequest("POST", url_.String(), strings.NewReader(multimap(params).Encode()))
if err != nil {
return err
}
// respect the environmnet's proxy settings
if prox_url, _ := http.ProxyFromEnvironment(hreq); prox_url != nil {
hreq.URL = prox_url
}
hreq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
hreq.Header.Set("X-Amz-Date", time.Now().UTC().Format(aws.ISO8601BasicFormat))
if s.Auth.Token() != "" {
hreq.Header.Set("X-Amz-Security-Token", s.Auth.Token())
}
signer := aws.NewV4Signer(s.Auth, "sqs", s.Region)
signer.Sign(hreq)
r, err := http.DefaultClient.Do(hreq)
if err != nil {
return err
}
defer r.Body.Close()
if debug {
dump, _ := httputil.DumpResponse(r, true)
log.Printf("DUMP:\n", string(dump))
}
if r.StatusCode != 200 {
return buildError(r)
}
err = xml.NewDecoder(r.Body).Decode(resp)
io.Copy(ioutil.Discard, r.Body)
return err
}
// getMetadataForgeModule queries the configured Puppet Forge and return
func getMetadataForgeModule(fm ForgeModule) ForgeModule {
baseUrl := config.Forge.Baseurl
if len(fm.baseUrl) > 0 {
baseUrl = fm.baseUrl
}
url := baseUrl + "/v3/releases/" + fm.author + "-" + fm.name + "-" + fm.version
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
req.Header.Set("Connection", "close")
proxyURL, err := http.ProxyFromEnvironment(req)
if err != nil {
Fatalf("getMetadataForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
before := time.Now()
Debugf("GETing " + url)
resp, err := client.Do(req)
duration := time.Since(before).Seconds()
Verbosef("GETing Forge metadata from " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
mutex.Lock()
syncForgeTime += duration
mutex.Unlock()
if err != nil {
Fatalf("getMetadataForgeModule(): Error while querying metadata for Forge module " + fm.name + " from " + url + ": " + err.Error())
}
defer resp.Body.Close()
if resp.Status == "200 OK" {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Fatalf("getMetadataForgeModule(): Error while reading response body for Forge module " + fm.name + " from " + url + ": " + err.Error())
}
before := time.Now()
currentRelease := gjson.Parse(string(body)).Map()
duration := time.Since(before).Seconds()
modulemd5sum := currentRelease["file_md5"].String()
moduleFilesize := currentRelease["file_size"].Int()
Debugf("module: " + fm.author + "/" + fm.name + " modulemd5sum: " + modulemd5sum + " moduleFilesize: " + strconv.FormatInt(moduleFilesize, 10))
mutex.Lock()
forgeJsonParseTime += duration
mutex.Unlock()
return ForgeModule{md5sum: modulemd5sum, fileSize: moduleFilesize}
} else {
Fatalf("getMetadataForgeModule(): Unexpected response code while GETing " + url + " " + resp.Status)
}
return ForgeModule{}
}
func NewMultipleHostReverseProxy(reg Registry) *httputil.ReverseProxy {
return &httputil.ReverseProxy{
Director: func(req *http.Request) {
req.URL.Scheme = "http"
req.URL.Host = req.Host
},
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return http.ProxyFromEnvironment(req)
},
Dial: func(network, addr string) (net.Conn, error) {
return loadBalance(network, addr, reg)
},
TLSHandshakeTimeout: 10 * time.Second,
},
}
}
func (this *curler) new_proxy() proxy_func {
if this.proxy_policy == CurlProxyPolicyAlwayseProxy {
return http.ProxyFromEnvironment
}
if this.proxy_policy == CurlProxyPolicyNoProxy {
return nil
}
if this.ruler != nil {
return func(req *http.Request) (*url.URL, error) {
uri := req.URL.String()
blocked := this.ruler.IsBlocked(uri)
if blocked {
return http.ProxyFromEnvironment(req)
}
return nil, nil
}
}
return nil
}
// NewMultipleHostReverseProxy creates a reverse proxy that will randomly
// select a host from the passed `targets`
func newPeterHostReverseProxy(target *url.URL) *httputil.ReverseProxy {
director := func(req *http.Request) {
fmt.Println("")
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = target.Path
fmt.Printf("DIRECTOR====> Scheme='%s' Host='%s' Path='%s'\n",
req.URL.Scheme, req.URL.Host, req.URL.Path)
os.Stdout.Sync()
}
return &httputil.ReverseProxy{
Director: director,
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
fmt.Printf("PROXY ====>req=%#v\n", req)
length := req.ContentLength
fmt.Printf("length=%d\n", length)
bodyReader := req.Body
body := readBody(bodyReader, int(length))
fmt.Printf("body=%v\n", body)
os.Stdout.Sync()
return http.ProxyFromEnvironment(req)
},
Dial: func(network, addr string) (net.Conn, error) {
fmt.Printf("DIAL ====>addr='%s'\n", addr)
conn, err := (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial(network, addr)
if err != nil {
println("Error during DIAL:", err.Error())
}
os.Stdout.Sync()
return conn, err
},
TLSHandshakeTimeout: 10 * time.Second,
},
}
}
// Creates a new Twitter client with the supplied OAuth configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
// export HTTP_PROXY=http://localhost:8888
//
// When using a proxy, disable TLS certificate verification with the following:
// export TLS_INSECURE=1
func NewClient(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client {
var (
host = "api.twitter.com"
base = "https://" + host
req, _ = http.NewRequest("GET", "https://api.twitter.com", nil)
proxy, _ = http.ProxyFromEnvironment(req)
transport *http.Transport
tlsconfig *tls.Config
)
if proxy != nil {
tlsconfig = &tls.Config{
InsecureSkipVerify: getEnvEitherCase("TLS_INSECURE") != "",
}
if tlsconfig.InsecureSkipVerify {
log.Printf("WARNING: SSL cert verification disabled\n")
}
transport = &http.Transport{
Proxy: http.ProxyURL(proxy),
TLSClientConfig: tlsconfig,
}
} else {
transport = &http.Transport{}
}
return &Client{
Host: host,
HttpClient: &http.Client{
Transport: transport,
},
User: user,
AppToken: nil,
OAuth: &oauth1a.Service{
RequestURL: base + "/oauth/request_token",
AuthorizeURL: base + "/oauth/authorize",
AccessURL: base + "/oauth/access_token",
ClientConfig: config,
Signer: new(oauth1a.HmacSha1Signer),
},
}
}
// Creates a new Infoblox client with the supplied user/pass configuration.
// Supports the use of HTTP proxies through the $HTTP_PROXY env var.
// For example:
// export HTTP_PROXY=http://localhost:8888
//
// When using a proxy, disable TLS certificate verification with the following:
// sslVerify = false
//
// To save and re-use infoblox session cookies, set useCookies = true
// NOTE: The infoblox cookie uses a comma separated string, and requires golang 1.3+ to be correctly stored.
//
func NewClient(host, username, password string, sslVerify, useCookies bool) *Client {
var (
req, _ = http.NewRequest("GET", host, nil)
proxy, _ = http.ProxyFromEnvironment(req)
transport *http.Transport
tlsconfig *tls.Config
)
tlsconfig = &tls.Config{
InsecureSkipVerify: !sslVerify,
}
if tlsconfig.InsecureSkipVerify {
log.Printf("WARNING: SSL cert verification disabled\n")
}
transport = &http.Transport{
TLSClientConfig: tlsconfig,
}
if proxy != nil {
transport.Proxy = http.ProxyURL(proxy)
}
client := &Client{
Host: host,
HttpClient: &http.Client{
Transport: transport,
},
Username: username,
Password: password,
UseCookies: useCookies,
}
if useCookies {
options := cookiejar.Options{
PublicSuffixList: publicsuffix.List,
}
jar, _ := cookiejar.New(&options)
client.HttpClient.Jar = jar
}
return client
}
func dialViaHTTPProxy(server string, port uint16) (*tls.Conn, error) {
xmppHost := fmt.Sprintf("%s:%d", server, port)
fakeRequest := http.Request{
URL: &url.URL{
Scheme: "https",
Host: xmppHost,
},
}
proxyURL, err := http.ProxyFromEnvironment(&fakeRequest)
if err != nil {
return nil, err
}
if proxyURL == nil {
return nil, nil
}
dialer := net.Dialer{
KeepAlive: netKeepAlive,
Timeout: netTimeout,
}
conn, err := dialer.Dial("tcp", proxyURL.Host)
if err != nil {
return nil, fmt.Errorf("Failed to connect to HTTP proxy server: %s", err)
}
fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", xmppHost, xmppHost)
response, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"})
if err != nil {
return nil, err
}
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Failed to connect to proxy: %s", response.Status)
}
return addTLS(server, conn)
}
func main() {
flag.Usage = usage
flag.Parse()
args := flag.Args()
if len(args) > 0 {
args = filter(args)
}
if ver {
fmt.Println("Version:", version)
os.Exit(2)
}
if printV != "A" && printV != "B" {
defaultSetting.DumpBody = false
}
var stdin []byte
if runtime.GOOS != "windows" {
fi, err := os.Stdin.Stat()
if err != nil {
panic(err)
}
if fi.Size() != 0 {
stdin, err = ioutil.ReadAll(os.Stdin)
if err != nil {
log.Fatal("Read from Stdin", err)
}
}
}
if *URL == "" {
usage()
}
if strings.HasPrefix(*URL, ":") {
urlb := []byte(*URL)
if *URL == ":" {
*URL = "http://localhost/"
} else if len(*URL) > 1 && urlb[1] != '/' {
*URL = "http://localhost" + *URL
} else {
*URL = "http://localhost" + string(urlb[1:])
}
}
if !strings.HasPrefix(*URL, "http://") && !strings.HasPrefix(*URL, "https://") {
*URL = "http://" + *URL
}
u, err := url.Parse(*URL)
if err != nil {
log.Fatal(err)
}
if auth != "" {
userpass := strings.Split(auth, ":")
if len(userpass) == 2 {
u.User = url.UserPassword(userpass[0], userpass[1])
} else {
u.User = url.User(auth)
}
}
*URL = u.String()
httpreq := getHTTP(*method, *URL, args)
if u.User != nil {
password, _ := u.User.Password()
httpreq.GetRequest().SetBasicAuth(u.User.Username(), password)
}
// Insecure SSL Support
if insecureSSL {
httpreq.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
// Proxy Support
if proxy != "" {
purl, err := url.Parse(proxy)
if err != nil {
log.Fatal("Proxy Url parse err", err)
}
httpreq.SetProxy(http.ProxyURL(purl))
} else {
eurl, err := http.ProxyFromEnvironment(httpreq.GetRequest())
if err != nil {
log.Fatal("Environment Proxy Url parse err", err)
}
httpreq.SetProxy(http.ProxyURL(eurl))
}
if body != "" {
httpreq.Body(body)
}
if len(stdin) > 0 {
var j interface{}
d := json.NewDecoder(bytes.NewReader(stdin))
d.UseNumber()
err = d.Decode(&j)
if err != nil {
httpreq.Body(stdin)
} else {
httpreq.JsonBody(j)
}
}
// AB bench
if bench {
httpreq.Debug(false)
RunBench(httpreq)
return
}
res, err := httpreq.Response()
if err != nil {
log.Fatalln("can't get the url", err)
}
// download file
if download {
var fl string
if disposition := res.Header.Get("Content-Disposition"); disposition != "" {
fls := strings.Split(disposition, ";")
for _, f := range fls {
f = strings.TrimSpace(f)
if strings.HasPrefix(f, "filename=") {
fl = strings.TrimLeft(f, "filename=")
}
}
}
if fl == "" {
_, fl = filepath.Split(u.Path)
}
fd, err := os.OpenFile(fl, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
log.Fatal("can't create file", err)
}
if runtime.GOOS != "windows" {
fmt.Println(Color(res.Proto, Magenta), Color(res.Status, Green))
for k, v := range res.Header {
fmt.Println(Color(k, Gray), ":", Color(strings.Join(v, " "), Cyan))
}
} else {
fmt.Println(res.Proto, res.Status)
for k, v := range res.Header {
fmt.Println(k, ":", strings.Join(v, " "))
}
}
fmt.Println("")
contentLength := res.Header.Get("Content-Length")
var total int64
if contentLength != "" {
total, _ = strconv.ParseInt(contentLength, 10, 64)
}
fmt.Printf("Downloading to \"%s\"\n", fl)
pb := NewProgressBar(total)
pb.Start()
multiWriter := io.MultiWriter(fd, pb)
_, err = io.Copy(multiWriter, res.Body)
if err != nil {
log.Fatal("Can't Write the body into file", err)
}
pb.Finish()
defer fd.Close()
defer res.Body.Close()
return
}
if runtime.GOOS != "windows" {
fi, err := os.Stdout.Stat()
if err != nil {
panic(err)
}
if fi.Mode()&os.ModeDevice == os.ModeDevice {
if printV == "A" || printV == "H" || printV == "B" {
dump := httpreq.DumpRequest()
if printV == "B" {
dps := strings.Split(string(dump), "\n")
for i, line := range dps {
if len(strings.Trim(line, "\r\n ")) == 0 {
dump = []byte(strings.Join(dps[i:], "\n"))
break
}
}
}
fmt.Println(ColorfulRequest(string(dump)))
fmt.Println("")
}
if printV == "A" || printV == "h" {
fmt.Println(Color(res.Proto, Magenta), Color(res.Status, Green))
for k, v := range res.Header {
fmt.Println(Color(k, Gray), ":", Color(strings.Join(v, " "), Cyan))
}
fmt.Println("")
}
if printV == "A" || printV == "b" {
body := formatResponseBody(res, httpreq, pretty)
fmt.Println(ColorfulResponse(body, res.Header.Get("Content-Type")))
}
} else {
body := formatResponseBody(res, httpreq, pretty)
_, err = os.Stdout.WriteString(body)
if err != nil {
log.Fatal(err)
}
}
} else {
if printV == "A" || printV == "H" || printV == "B" {
dump := httpreq.DumpRequest()
if printV == "B" {
dps := strings.Split(string(dump), "\n")
for i, line := range dps {
if len(strings.Trim(line, "\r\n ")) == 0 {
dump = []byte(strings.Join(dps[i:], "\n"))
break
}
}
}
fmt.Println(string(dump))
fmt.Println("")
}
if printV == "A" || printV == "h" {
fmt.Println(res.Proto, res.Status)
for k, v := range res.Header {
fmt.Println(k, ":", strings.Join(v, " "))
}
fmt.Println("")
}
if printV == "A" || printV == "b" {
body := formatResponseBody(res, httpreq, pretty)
fmt.Println(body)
}
}
}
func SmartProxy(req *http.Request) (*url.URL, error) {
return http.ProxyFromEnvironment(req)
}
func downloadForgeModule(name string, version string, fm ForgeModule, retryCount int) {
funcName := funcName()
var wgForgeModule sync.WaitGroup
extractR, extractW := io.Pipe()
saveFileR, saveFileW := io.Pipe()
//url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz"
fileName := name + "-" + version + ".tar.gz"
if !fileExists(config.ForgeCacheDir + name + "-" + version) {
baseUrl := config.Forge.Baseurl
if len(fm.baseUrl) > 0 {
baseUrl = fm.baseUrl
}
url := baseUrl + "/v3/files/" + fileName
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
req.Header.Set("Connection", "close")
proxyURL, err := http.ProxyFromEnvironment(req)
if err != nil {
Fatalf(funcName + "(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
before := time.Now()
Debugf("GETing " + url)
resp, err := client.Do(req)
duration := time.Since(before).Seconds()
Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
mutex.Lock()
syncForgeTime += duration
mutex.Unlock()
if err != nil {
Fatalf(funcName + "(): Error while GETing Forge module " + name + " from " + url + ": " + err.Error())
}
defer resp.Body.Close()
if resp.Status == "200 OK" {
wgForgeModule.Add(1)
go func() {
defer wgForgeModule.Done()
Debugf(funcName + "(): Trying to create " + config.ForgeCacheDir + fileName)
out, err := os.Create(config.ForgeCacheDir + fileName)
if err != nil {
Fatalf(funcName + "(): Error while creating file for Forge module " + config.ForgeCacheDir + fileName + " Error: " + err.Error())
}
defer out.Close()
io.Copy(out, saveFileR)
Debugf(funcName + "(): Finished creating " + config.ForgeCacheDir + fileName)
}()
wgForgeModule.Add(1)
go extractForgeModule(&wgForgeModule, extractR, fileName)
wgForgeModule.Add(1)
go func() {
defer wgForgeModule.Done()
// after completing the copy, we need to close
// the PipeWriters to propagate the EOF to all
// PipeReaders to avoid deadlock
defer extractW.Close()
defer saveFileW.Close()
var mw io.Writer
// build the multiwriter for all the pipes
mw = io.MultiWriter(extractW, saveFileW)
// copy the data into the multiwriter
if _, err := io.Copy(mw, resp.Body); err != nil {
Fatalf("Error while writing to MultiWriter " + err.Error())
}
}()
} else {
Fatalf(funcName + "(): Unexpected response code while GETing " + url + " " + resp.Status)
}
} else {
Debugf("Using cache for Forge module " + name + " version: " + version)
}
wgForgeModule.Wait()
if checkSum || fm.sha256sum != "" {
fm.version = version
if doForgeModuleIntegrityCheck(fm) {
if retryCount == 0 {
Fatalf("downloadForgeModule(): giving up for Puppet module " + name + " version: " + version)
}
Warnf("Retrying...")
purgeDir(config.ForgeCacheDir+fileName, "downloadForgeModule()")
purgeDir(strings.Replace(config.ForgeCacheDir+fileName, ".tar.gz", "/", -1), "downloadForgeModule()")
// retry if hash sum mismatch found
downloadForgeModule(name, version, fm, retryCount-1)
}
}
}
func initMQ(orig_ctx Context, try_proxy bool, proxy string) (ctx Context, err error) {
ctx = orig_ctx
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("initMQ() -> %v", e)
}
ctx.Channels.Log <- mig.Log{Desc: "leaving initMQ()"}.Debug()
}()
//Define the AMQP binding
ctx.MQ.Bind.Queue = fmt.Sprintf("mig.agt.%s", ctx.Agent.QueueLoc)
ctx.MQ.Bind.Key = fmt.Sprintf("mig.agt.%s", ctx.Agent.QueueLoc)
// parse the dial string and use TLS if using amqps
amqp_uri, err := amqp.ParseURI(AMQPBROKER)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("AMQP: host=%s, port=%d, vhost=%s", amqp_uri.Host, amqp_uri.Port, amqp_uri.Vhost)}.Debug()
if amqp_uri.Scheme == "amqps" {
ctx.MQ.UseTLS = true
}
// create an AMQP configuration with specific timers
var dialConfig amqp.Config
dialConfig.Heartbeat = 2 * ctx.Sleeper
if try_proxy {
// if in try_proxy mode, the agent will try to connect to the relay using a CONNECT proxy
// but because CONNECT is a HTTP method, not available in AMQP, we need to establish
// that connection ourselves, and give it back to the amqp.DialConfig method
if proxy == "" {
// try to get the proxy from the environemnt (variable HTTP_PROXY)
target := "http://" + amqp_uri.Host + ":" + fmt.Sprintf("%d", amqp_uri.Port)
req, err := http.NewRequest("GET", target, nil)
if err != nil {
panic(err)
}
proxy_url, err := http.ProxyFromEnvironment(req)
if err != nil {
panic(err)
}
if proxy_url == nil {
panic("Failed to find a suitable proxy in environment")
}
proxy = proxy_url.Host
ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Found proxy at %s", proxy)}.Debug()
}
ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Connecting via proxy %s", proxy)}.Debug()
dialConfig.Dial = func(network, addr string) (conn net.Conn, err error) {
// connect to the proxy
conn, err = net.DialTimeout("tcp", proxy, 5*time.Second)
if err != nil {
return
}
// write a CONNECT request in the tcp connection
fmt.Fprintf(conn, "CONNECT "+addr+" HTTP/1.1\r\nHost: "+addr+"\r\n\r\n")
// verify status is 200, and flush the buffer
status, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
return
}
if status == "" || len(status) < 12 {
err = fmt.Errorf("Invalid status received from proxy: '%s'", status[0:len(status)-2])
return
}
// 9th character in response should be "2"
// HTTP/1.0 200 Connection established
// ^
if status[9] != '2' {
err = fmt.Errorf("Invalid status received from proxy: '%s'", status[0:len(status)-2])
return
}
ctx.Agent.Env.IsProxied = true
ctx.Agent.Env.Proxy = proxy
return
}
} else {
dialConfig.Dial = func(network, addr string) (net.Conn, error) {
return net.DialTimeout(network, addr, 5*time.Second)
}
}
if ctx.MQ.UseTLS {
ctx.Channels.Log <- mig.Log{Desc: "Loading AMQPS TLS parameters"}.Debug()
// import the client certificates
cert, err := tls.X509KeyPair(AGENTCERT, AGENTKEY)
if err != nil {
panic(err)
}
// import the ca cert
ca := x509.NewCertPool()
if ok := ca.AppendCertsFromPEM(CACERT); !ok {
panic("failed to import CA Certificate")
}
TLSconfig := tls.Config{Certificates: []tls.Certificate{cert},
RootCAs: ca,
InsecureSkipVerify: false,
Rand: rand.Reader}
dialConfig.TLSClientConfig = &TLSconfig
}
// Open AMQP connection
ctx.Channels.Log <- mig.Log{Desc: "Establishing connection to relay"}.Debug()
ctx.MQ.conn, err = amqp.DialConfig(AMQPBROKER, dialConfig)
if err != nil {
ctx.Channels.Log <- mig.Log{Desc: "Connection failed"}.Debug()
panic(err)
}
ctx.MQ.Chan, err = ctx.MQ.conn.Channel()
if err != nil {
panic(err)
}
// Limit the number of message the channel will receive at once
err = ctx.MQ.Chan.Qos(1, // prefetch count (in # of msg)
0, // prefetch size (in bytes)
false) // is global
_, err = ctx.MQ.Chan.QueueDeclare(ctx.MQ.Bind.Queue, // Queue name
true, // is durable
false, // is autoDelete
false, // is exclusive
false, // is noWait
nil) // AMQP args
if err != nil {
panic(err)
}
err = ctx.MQ.Chan.QueueBind(ctx.MQ.Bind.Queue, // Queue name
ctx.MQ.Bind.Key, // Routing key name
mig.Mq_Ex_ToAgents, // Exchange name
false, // is noWait
nil) // AMQP args
if err != nil {
panic(err)
}
// Consume AMQP message into channel
ctx.MQ.Bind.Chan, err = ctx.MQ.Chan.Consume(ctx.MQ.Bind.Queue, // queue name
"", // some tag
false, // is autoAck
false, // is exclusive
false, // is noLocal
false, // is noWait
nil) // AMQP args
if err != nil {
panic(err)
}
return
}
func queryForgeApi(name string, file string) ForgeResult {
//url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1)
url := "https://forgeapi.puppetlabs.com:443/v3/modules?query=" + name
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal("queryForgeApi(): Error creating GET request for Puppetlabs forge API", err)
os.Exit(1)
}
if fileInfo, err := os.Stat(file); err == nil {
Debugf("queryForgeApi(): adding If-Modified-Since:" + string(fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + " to Forge query")
req.Header.Set("If-Modified-Since", fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
}
req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
req.Header.Set("Connection", "close")
proxyUrl, err := http.ProxyFromEnvironment(req)
if err != nil {
log.Fatal("queryForgeApi(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err)
os.Exit(1)
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
before := time.Now()
resp, err := client.Do(req)
duration := time.Since(before).Seconds()
Verbosef("Querying Forge API " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
mutex.Lock()
syncForgeTime += duration
mutex.Unlock()
if err != nil {
panic(err)
}
defer resp.Body.Close()
if resp.Status == "200 OK" {
// need to get latest version
body, err := ioutil.ReadAll(resp.Body)
//fmt.Println(string(body))
reCurrent := regexp.MustCompile("\\s*\"current_release\": {\n\\s*\"uri\": \"([^\"]+)\",")
if m := reCurrent.FindStringSubmatch(string(body)); len(m) > 1 {
//fmt.Println(m[1])
if strings.Count(m[1], "-") < 2 {
log.Fatal("queryForgeApi(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", m[1], " should contain three '-' characters")
os.Exit(1)
} else {
version := strings.Split(m[1], "-")[2]
Debugf("queryForgeApi(): found version " + version + " for " + name + "-latest")
mutex.Lock()
latestForgeModules[name] = version
mutex.Unlock()
return ForgeResult{true, version}
}
}
if err != nil {
panic(err)
}
return ForgeResult{false, ""}
} else if resp.Status == "304 Not Modified" {
Debugf("queryForgeApi(): Got 304 nothing to do for module " + name)
return ForgeResult{false, ""}
} else {
Debugf("queryForgeApi(): Unexpected response code " + resp.Status)
return ForgeResult{false, ""}
}
}
// RestAPICall - general rest method caller
func (c *Client) RestAPICall(method Method, path string, options interface{}) ([]byte, error) {
log.Debugf("RestAPICall %s - %s%s", method, utils.Sanatize(c.Endpoint), path)
var (
Url *url.URL
err error
req *http.Request
)
Url, err = url.Parse(utils.Sanatize(c.Endpoint))
if err != nil {
return nil, err
}
Url.Path += path
// Manage the query string
c.GetQueryString(Url)
log.Debugf("*** url => %s", Url.String())
log.Debugf("*** method => %s", method.String())
// parse url
reqUrl, err := url.Parse(Url.String())
if err != nil {
return nil, fmt.Errorf("Error with request: %v - %q", Url, err)
}
// handle options
if options != nil {
OptionsJSON, err := json.Marshal(options)
if err != nil {
return nil, err
}
log.Debugf("*** options => %+v", bytes.NewBuffer(OptionsJSON))
req, err = http.NewRequest(method.String(), reqUrl.String(), bytes.NewBuffer(OptionsJSON))
} else {
req, err = http.NewRequest(method.String(), reqUrl.String(), nil)
}
if err != nil {
return nil, fmt.Errorf("Error with request: %v - %q", Url, err)
}
// setup proxy
proxyUrl, err := http.ProxyFromEnvironment(req)
if err != nil {
return nil, fmt.Errorf("Error with proxy: %v - %q", proxyUrl, err)
}
if proxyUrl != nil {
tr.Proxy = http.ProxyURL(proxyUrl)
log.Debugf("*** proxy => %+v", tr.Proxy)
}
// build the auth headerU
for k, v := range c.Option.Headers {
log.Debugf("Headers -> %s -> %+v\n", k, v)
req.Header.Add(k, v)
}
// req.SetBasicAuth(c.User, c.APIKey)
req.Method = fmt.Sprintf("%s", method.String())
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// TODO: CLeanup Later
// DEBUGGING WHILE WE WORK
// DEBUGGING WHILE WE WORK
// fmt.Printf("METHOD --> %+v\n",method)
log.Debugf("REQ --> %+v\n", req)
log.Debugf("RESP --> %+v\n", resp)
log.Debugf("ERROR --> %+v\n", err)
// DEBUGGING WHILE WE WORK
data, err := ioutil.ReadAll(resp.Body)
if !c.isOkStatus(resp.StatusCode) {
type apiErr struct {
Err string `json:"details"`
}
var outErr apiErr
json.Unmarshal(data, &outErr)
return nil, fmt.Errorf("Error in response: %s\n Response Status: %s", outErr.Err, resp.Status)
}
if err != nil {
return nil, err
}
return data, nil
}
func downloadForgeModule(name string, version string) {
//url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz"
fileName := name + "-" + version + ".tar.gz"
if _, err := os.Stat(config.ForgeCacheDir + name + "-" + version); os.IsNotExist(err) {
url := "https://forgeapi.puppetlabs.com/v3/files/" + fileName
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
req.Header.Set("Connection", "close")
proxyUrl, err := http.ProxyFromEnvironment(req)
if err != nil {
log.Fatal("downloadForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err)
os.Exit(1)
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
before := time.Now()
resp, err := client.Do(req)
duration := time.Since(before).Seconds()
Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
mutex.Lock()
syncForgeTime += duration
mutex.Unlock()
if err != nil {
log.Print("downloadForgeModule(): Error while GETing Forge module ", name, " from ", url, ": ", err)
os.Exit(1)
}
defer resp.Body.Close()
if resp.Status == "200 OK" {
Debugf("downloadForgeModule(): Trying to create " + config.ForgeCacheDir + fileName)
out, err := os.Create(config.ForgeCacheDir + fileName)
if err != nil {
log.Print("downloadForgeModule(): Error while creating file for Forge module "+config.ForgeCacheDir+fileName, err)
os.Exit(1)
}
defer out.Close()
io.Copy(out, resp.Body)
file, err := os.Open(config.ForgeCacheDir + fileName)
if err != nil {
fmt.Println("downloadForgeModule(): Error while opening file", file, err)
os.Exit(1)
}
defer file.Close()
var fileReader io.ReadCloser = resp.Body
if strings.HasSuffix(fileName, ".gz") {
if fileReader, err = pgzip.NewReader(file); err != nil {
fmt.Println("downloadForgeModule(): pgzip reader error for module ", fileName, " error:", err)
os.Exit(1)
}
defer fileReader.Close()
}
tarBallReader := tar.NewReader(fileReader)
if err = os.Chdir(config.ForgeCacheDir); err != nil {
fmt.Println("downloadForgeModule(): error while chdir to", config.ForgeCacheDir, err)
os.Exit(1)
}
for {
header, err := tarBallReader.Next()
if err != nil {
if err == io.EOF {
break
}
fmt.Println("downloadForgeModule(): error while tar reader.Next() for ", fileName, err)
os.Exit(1)
}
// get the individual filename and extract to the current directory
filename := header.Name
//Debugf("downloadForgeModule(): Trying to extract file" + filename)
switch header.Typeflag {
case tar.TypeDir:
// handle directory
//fmt.Println("Creating directory :", filename)
//err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755 if you prefer
err = os.MkdirAll(filename, os.FileMode(0755)) // or use 0755 if you prefer
if err != nil {
fmt.Println("downloadForgeModule(): error while MkdirAll()", filename, err)
os.Exit(1)
}
case tar.TypeReg:
// handle normal file
//fmt.Println("Untarring :", filename)
writer, err := os.Create(filename)
if err != nil {
fmt.Println("downloadForgeModule(): error while Create()", filename, err)
os.Exit(1)
}
io.Copy(writer, tarBallReader)
err = os.Chmod(filename, os.FileMode(0644))
if err != nil {
fmt.Println("downloadForgeModule(): error while Chmod()", filename, err)
os.Exit(1)
}
writer.Close()
default:
fmt.Printf("Unable to untar type : %c in file %s", header.Typeflag, filename)
}
}
} else {
log.Print("downloadForgeModule(): Unexpected response code while GETing " + url + resp.Status)
os.Exit(1)
}
} else {
Debugf("downloadForgeModule(): Using cache for Forge module " + name + " version: " + version)
}
}
// websocketConn establishes a connection to the given URL, respecting any proxy configuration in the environment.
// A standard proxying setup involves setting the following environment variables
// (may be listed in /etc/ecs/ecs.config if using ecs-init):
// HTTP_PROXY=http://<your-proxy>/ # HTTPS_PROXY may be set instead or additionally
// NO_PROXY=169.254.169.254,/var/run/docker.sock # Directly connect to metadata service and docker socket
func (cs *ClientServerImpl) websocketConn(parsedURL *url.URL, request *http.Request) (net.Conn, error) {
proxyURL, err := http.ProxyFromEnvironment(request)
if err != nil {
return nil, err
}
// url.Host might not have the port, but tls.Dial needs it
targetHost := parsedURL.Host
if !strings.Contains(targetHost, ":") {
targetHost += ":443"
}
targetHostname, _, err := net.SplitHostPort(targetHost)
if err != nil {
return nil, err
}
tlsConfig := tls.Config{ServerName: targetHostname, InsecureSkipVerify: cs.AcceptInvalidCert}
timeoutDialer := &net.Dialer{Timeout: wsConnectTimeout}
if proxyURL == nil {
// directly connect
log.Info("Creating poll dialer", "host", parsedURL.Host)
return tls.DialWithDialer(timeoutDialer, "tcp", targetHost, &tlsConfig)
}
// connect via proxy
log.Info("Creating poll dialer", "proxy", proxyURL.Host, "host", parsedURL.Host)
plainConn, err := timeoutDialer.Dial("tcp", proxyURL.Host)
if err != nil {
return nil, err
}
// TLS over an HTTP proxy via CONNECT taken from: https://golang.org/src/net/http/transport.go
connectReq := &http.Request{
Method: "CONNECT",
URL: &url.URL{Opaque: targetHost},
Host: targetHost,
Header: make(http.Header),
}
if proxyUser := proxyURL.User; proxyUser != nil {
username := proxyUser.Username()
password, _ := proxyUser.Password()
auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
connectReq.Header.Set("Proxy-Authorization", "Basic "+auth)
}
connectReq.Write(plainConn)
// Read response.
// Okay to use and discard buffered reader here, because
// TLS server will not speak until spoken to.
br := bufio.NewReader(plainConn)
resp, err := http.ReadResponse(br, connectReq)
if err != nil {
plainConn.Close()
return nil, err
}
if resp.StatusCode != 200 {
plainConn.Close()
return nil, errors.New(resp.Status)
}
tlsConn := tls.Client(plainConn, &tlsConfig)
return tlsConn, nil
}
func queryForgeAPI(name string, file string, fm ForgeModule) ForgeResult {
//url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1)
baseUrl := config.Forge.Baseurl
if len(fm.baseUrl) > 0 {
baseUrl = fm.baseUrl
}
//url := baseUrl + "/v3/modules?query=" + name
url := baseUrl + "/v3/releases?module=" + name + "&owner=" + fm.author + "&sort_by=release_date&limit=1"
req, err := http.NewRequest("GET", url, nil)
if err != nil {
Fatalf("queryForgeAPI(): Error creating GET request for Puppetlabs forge API" + err.Error())
}
if fileInfo, err := os.Stat(file); err == nil {
Debugf("adding If-Modified-Since:" + string(fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + " to Forge query")
req.Header.Set("If-Modified-Since", fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
}
req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
req.Header.Set("Connection", "close")
proxyURL, err := http.ProxyFromEnvironment(req)
if err != nil {
Fatalf("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
before := time.Now()
resp, err := client.Do(req)
if err != nil {
Fatalf("queryForgeAPI(): Error while issuing the HTTP request to " + url + " Error: " + err.Error())
}
duration := time.Since(before).Seconds()
Verbosef("Querying Forge API " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
mutex.Lock()
syncForgeTime += duration
mutex.Unlock()
defer resp.Body.Close()
if resp.Status == "200 OK" {
// need to get latest version
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Fatalf("queryForgeAPI(): Error while reading response body for Forge module " + fm.name + " from " + url + ": " + err.Error())
}
before := time.Now()
currentRelease := gjson.Get(string(body), "results.0").Map()
duration := time.Since(before).Seconds()
version := currentRelease["version"].String()
modulemd5sum := currentRelease["file_md5"].String()
moduleFilesize := currentRelease["file_size"].Int()
mutex.Lock()
forgeJsonParseTime += duration
mutex.Unlock()
Debugf("found version " + version + " for " + name + "-latest")
latestForgeModules.Lock()
latestForgeModules.m[name] = version
latestForgeModules.Unlock()
lastCheckedFile := config.ForgeCacheDir + name + "-latest-last-checked"
Debugf("writing last-checked file " + lastCheckedFile)
f, _ := os.Create(lastCheckedFile)
defer f.Close()
return ForgeResult{true, version, modulemd5sum, moduleFilesize}
} else if resp.Status == "304 Not Modified" {
Debugf("Got 304 nothing to do for module " + name)
return ForgeResult{false, "", "", 0}
} else {
Debugf("Unexpected response code " + resp.Status)
return ForgeResult{false, "", "", 0}
}
}