// proxySettings will default to the default proxy settings if none are provided // if settings are provided – they will override the environment variables func (ro RequestOptions) proxySettings(req *http.Request) (*url.URL, error) { // No proxies – lets use the default if len(ro.Proxies) == 0 { return http.ProxyFromEnvironment(req) } // There was a proxy specified – do we support the protocol? if _, ok := ro.Proxies[req.URL.Scheme]; ok { return ro.Proxies[req.URL.Scheme], nil } // Proxies were specified but not for any protocol that we use return http.ProxyFromEnvironment(req) }
// Creates a new Twitter client with the supplied OAuth configuration. // Supports the use of HTTP proxies through the $HTTP_PROXY env var. // For example: // export HTTP_PROXY=http://localhost:8888 func NewClient(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client { var ( host = "api.twitter.com" base = "https://" + host req, _ = http.NewRequest("GET", "https://api.twitter.com", nil) proxy, _ = http.ProxyFromEnvironment(req) transport *http.Transport ) if proxy != nil { transport = &http.Transport{ Proxy: http.ProxyURL(proxy), } } else { transport = &http.Transport{} } return &Client{ Host: host, HttpClient: &http.Client{ Transport: transport, }, User: user, AppToken: nil, OAuth: &oauth1a.Service{ RequestURL: base + "/oauth/request_token", AuthorizeURL: base + "/oauth/authorize", AccessURL: base + "/oauth/access_token", ClientConfig: config, Signer: new(oauth1a.HmacSha1Signer), }, } }
// NewClient creates a new Client object. // The token can be empty if you plan on creating // the token using the `CreateToken` func. If you want // to use your existing token, you need to pass it as the paramter. // It returns a Client object that is pre-configured for usage. func NewClient(token string) *Client { // Perhaps we should allow people to configure this through some // variable. It would make the package almost 100% backwards compatible // with the public api (https://developer.kwikdesk.com) var ( host = "platform.kwikdesk.com" base = "https://" + host req, _ = http.NewRequest("GET", base, nil) proxy, _ = http.ProxyFromEnvironment(req) transport *http.Transport ) transport = &http.Transport{} if proxy != nil { transport = &http.Transport{ Proxy: http.ProxyURL(proxy), } } return &Client{ Host: host, FullHost: base, HttpClient: &http.Client{ Transport: transport, }, ContentType: kdHTTPContentType, XToken: token, } }
// Creates a new Infoblox client with the supplied user/pass configuration. // Supports the use of HTTP proxies through the $HTTP_PROXY env var. // For example: // export HTTP_PROXY=http://localhost:8888 // // When using a proxy, disable TLS certificate verification with the following: // sslVerify = false func NewClient(host, username, password string, sslVerify bool) *Client { var ( req, _ = http.NewRequest("GET", host, nil) proxy, _ = http.ProxyFromEnvironment(req) transport *http.Transport tlsconfig *tls.Config ) tlsconfig = &tls.Config{ InsecureSkipVerify: !sslVerify, } if tlsconfig.InsecureSkipVerify { fmt.Println("WARNING: SSL cert verification disabled!") } transport = &http.Transport{ TLSClientConfig: tlsconfig, } if proxy != nil { transport.Proxy = http.ProxyURL(proxy) } return &Client{ Host: host, HttpClient: &http.Client{ Transport: transport, }, Username: username, Password: password, } }
func getProxy() *url.URL { req, err := http.NewRequest("GET", "https://api.heroku.com", nil) PrintError(err, false) proxy, err := http.ProxyFromEnvironment(req) PrintError(err, false) return proxy }
// NewMultipleHostReverseProxy creates a reverse proxy that will randomly // select a host from the passed `targets` func NewMultipleHostReverseProxy(targets []*url.URL) *httputil.ReverseProxy { director := func(req *http.Request) { println("CALLING DIRECTOR") target := targets[rand.Int()%len(targets)] req.URL.Scheme = target.Scheme req.URL.Host = target.Host req.URL.Path = target.Path } return &httputil.ReverseProxy{ Director: director, Transport: &http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { println("CALLING PROXY") return http.ProxyFromEnvironment(req) }, Dial: func(network, addr string) (net.Conn, error) { println("CALLING DIAL") conn, err := (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).Dial(network, addr) if err != nil { println("Error during DIAL:", err.Error()) } return conn, err }, TLSHandshakeTimeout: 10 * time.Second, }, } }
func (c *Client) SetCertificate(cert tls.Certificate) { t := c.Client.Transport.(*http.Transport) // Extension certificate t.TLSClientConfig.Certificates = []tls.Certificate{cert} // Proxy to vCenter host on port 80 host, _ := splitHostPort(c.u.Host) // Should be no reason to change the default port other than testing port := os.Getenv("GOVC_TUNNEL_PROXY_PORT") if port != "" { host += ":" + port } c.p = &url.URL{ Scheme: "http", Host: host, } t.Proxy = func(r *http.Request) (*url.URL, error) { // Only sdk requests should be proxied if r.URL.Path == "/sdk" { return c.p, nil } return http.ProxyFromEnvironment(r) } // Rewrite url Host to use the sdk tunnel, required for a certificate request. c.u.Host = sdkTunnel }
// dial dials the host specified by req, using TLS if appropriate, optionally // using a proxy server if one is configured via environment variables. func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) { proxyURL, err := http.ProxyFromEnvironment(req) if err != nil { return nil, err } if proxyURL == nil { return s.dialWithoutProxy(req.URL) } // ensure we use a canonical host with proxyReq targetHost := netutil.CanonicalAddr(req.URL) // proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support proxyReq := http.Request{ Method: "CONNECT", URL: &url.URL{}, Host: targetHost, } proxyDialConn, err := s.dialWithoutProxy(proxyURL) if err != nil { return nil, err } proxyClientConn := httputil.NewProxyClientConn(proxyDialConn, nil) _, err = proxyClientConn.Do(&proxyReq) if err != nil && err != httputil.ErrPersistEOF { return nil, err } rwc, _ := proxyClientConn.Hijack() if req.URL.Scheme != "https" { return rwc, nil } host, _, err := net.SplitHostPort(req.URL.Host) if err != nil { return nil, err } if len(s.tlsConfig.ServerName) == 0 { s.tlsConfig.ServerName = host } tlsConn := tls.Client(rwc, s.tlsConfig) // need to manually call Handshake() so we can call VerifyHostname() below if err := tlsConn.Handshake(); err != nil { return nil, err } if err := tlsConn.VerifyHostname(host); err != nil { return nil, err } return tlsConn, nil }
func getProxy(req *http.Request) (*url.URL, error) { if config.ProxyServer != "" { u, err := url.Parse(config.ProxyServer) if err == nil { return u, nil } } return http.ProxyFromEnvironment(req) }
func (s *SQS) query(queueUrl string, params map[string]string, resp interface{}) (err error) { var url_ *url.URL if queueUrl != "" && len(queueUrl) > len(s.Region.SQSEndpoint) { url_, err = url.Parse(queueUrl) } else { url_, err = url.Parse(s.Region.SQSEndpoint) } if err != nil { return err } params["Version"] = "2012-11-05" hreq, err := http.NewRequest("POST", url_.String(), strings.NewReader(multimap(params).Encode())) if err != nil { return err } // respect the environmnet's proxy settings if prox_url, _ := http.ProxyFromEnvironment(hreq); prox_url != nil { hreq.URL = prox_url } hreq.Header.Set("Content-Type", "application/x-www-form-urlencoded") hreq.Header.Set("X-Amz-Date", time.Now().UTC().Format(aws.ISO8601BasicFormat)) if s.Auth.Token() != "" { hreq.Header.Set("X-Amz-Security-Token", s.Auth.Token()) } signer := aws.NewV4Signer(s.Auth, "sqs", s.Region) signer.Sign(hreq) r, err := http.DefaultClient.Do(hreq) if err != nil { return err } defer r.Body.Close() if debug { dump, _ := httputil.DumpResponse(r, true) log.Printf("DUMP:\n", string(dump)) } if r.StatusCode != 200 { return buildError(r) } err = xml.NewDecoder(r.Body).Decode(resp) io.Copy(ioutil.Discard, r.Body) return err }
// getMetadataForgeModule queries the configured Puppet Forge and return func getMetadataForgeModule(fm ForgeModule) ForgeModule { baseUrl := config.Forge.Baseurl if len(fm.baseUrl) > 0 { baseUrl = fm.baseUrl } url := baseUrl + "/v3/releases/" + fm.author + "-" + fm.name + "-" + fm.version req, err := http.NewRequest("GET", url, nil) req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/") req.Header.Set("Connection", "close") proxyURL, err := http.ProxyFromEnvironment(req) if err != nil { Fatalf("getMetadataForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error()) } client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} before := time.Now() Debugf("GETing " + url) resp, err := client.Do(req) duration := time.Since(before).Seconds() Verbosef("GETing Forge metadata from " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s") mutex.Lock() syncForgeTime += duration mutex.Unlock() if err != nil { Fatalf("getMetadataForgeModule(): Error while querying metadata for Forge module " + fm.name + " from " + url + ": " + err.Error()) } defer resp.Body.Close() if resp.Status == "200 OK" { body, err := ioutil.ReadAll(resp.Body) if err != nil { Fatalf("getMetadataForgeModule(): Error while reading response body for Forge module " + fm.name + " from " + url + ": " + err.Error()) } before := time.Now() currentRelease := gjson.Parse(string(body)).Map() duration := time.Since(before).Seconds() modulemd5sum := currentRelease["file_md5"].String() moduleFilesize := currentRelease["file_size"].Int() Debugf("module: " + fm.author + "/" + fm.name + " modulemd5sum: " + modulemd5sum + " moduleFilesize: " + strconv.FormatInt(moduleFilesize, 10)) mutex.Lock() forgeJsonParseTime += duration mutex.Unlock() return ForgeModule{md5sum: modulemd5sum, fileSize: moduleFilesize} } else { Fatalf("getMetadataForgeModule(): Unexpected response code while GETing " + url + " " + resp.Status) } return ForgeModule{} }
func NewMultipleHostReverseProxy(reg Registry) *httputil.ReverseProxy { return &httputil.ReverseProxy{ Director: func(req *http.Request) { req.URL.Scheme = "http" req.URL.Host = req.Host }, Transport: &http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { return http.ProxyFromEnvironment(req) }, Dial: func(network, addr string) (net.Conn, error) { return loadBalance(network, addr, reg) }, TLSHandshakeTimeout: 10 * time.Second, }, } }
func (this *curler) new_proxy() proxy_func { if this.proxy_policy == CurlProxyPolicyAlwayseProxy { return http.ProxyFromEnvironment } if this.proxy_policy == CurlProxyPolicyNoProxy { return nil } if this.ruler != nil { return func(req *http.Request) (*url.URL, error) { uri := req.URL.String() blocked := this.ruler.IsBlocked(uri) if blocked { return http.ProxyFromEnvironment(req) } return nil, nil } } return nil }
// NewMultipleHostReverseProxy creates a reverse proxy that will randomly // select a host from the passed `targets` func newPeterHostReverseProxy(target *url.URL) *httputil.ReverseProxy { director := func(req *http.Request) { fmt.Println("") req.URL.Scheme = target.Scheme req.URL.Host = target.Host req.URL.Path = target.Path fmt.Printf("DIRECTOR====> Scheme='%s' Host='%s' Path='%s'\n", req.URL.Scheme, req.URL.Host, req.URL.Path) os.Stdout.Sync() } return &httputil.ReverseProxy{ Director: director, Transport: &http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { fmt.Printf("PROXY ====>req=%#v\n", req) length := req.ContentLength fmt.Printf("length=%d\n", length) bodyReader := req.Body body := readBody(bodyReader, int(length)) fmt.Printf("body=%v\n", body) os.Stdout.Sync() return http.ProxyFromEnvironment(req) }, Dial: func(network, addr string) (net.Conn, error) { fmt.Printf("DIAL ====>addr='%s'\n", addr) conn, err := (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).Dial(network, addr) if err != nil { println("Error during DIAL:", err.Error()) } os.Stdout.Sync() return conn, err }, TLSHandshakeTimeout: 10 * time.Second, }, } }
// Creates a new Twitter client with the supplied OAuth configuration. // Supports the use of HTTP proxies through the $HTTP_PROXY env var. // For example: // export HTTP_PROXY=http://localhost:8888 // // When using a proxy, disable TLS certificate verification with the following: // export TLS_INSECURE=1 func NewClient(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client { var ( host = "api.twitter.com" base = "https://" + host req, _ = http.NewRequest("GET", "https://api.twitter.com", nil) proxy, _ = http.ProxyFromEnvironment(req) transport *http.Transport tlsconfig *tls.Config ) if proxy != nil { tlsconfig = &tls.Config{ InsecureSkipVerify: getEnvEitherCase("TLS_INSECURE") != "", } if tlsconfig.InsecureSkipVerify { log.Printf("WARNING: SSL cert verification disabled\n") } transport = &http.Transport{ Proxy: http.ProxyURL(proxy), TLSClientConfig: tlsconfig, } } else { transport = &http.Transport{} } return &Client{ Host: host, HttpClient: &http.Client{ Transport: transport, }, User: user, AppToken: nil, OAuth: &oauth1a.Service{ RequestURL: base + "/oauth/request_token", AuthorizeURL: base + "/oauth/authorize", AccessURL: base + "/oauth/access_token", ClientConfig: config, Signer: new(oauth1a.HmacSha1Signer), }, } }
// Creates a new Infoblox client with the supplied user/pass configuration. // Supports the use of HTTP proxies through the $HTTP_PROXY env var. // For example: // export HTTP_PROXY=http://localhost:8888 // // When using a proxy, disable TLS certificate verification with the following: // sslVerify = false // // To save and re-use infoblox session cookies, set useCookies = true // NOTE: The infoblox cookie uses a comma separated string, and requires golang 1.3+ to be correctly stored. // func NewClient(host, username, password string, sslVerify, useCookies bool) *Client { var ( req, _ = http.NewRequest("GET", host, nil) proxy, _ = http.ProxyFromEnvironment(req) transport *http.Transport tlsconfig *tls.Config ) tlsconfig = &tls.Config{ InsecureSkipVerify: !sslVerify, } if tlsconfig.InsecureSkipVerify { log.Printf("WARNING: SSL cert verification disabled\n") } transport = &http.Transport{ TLSClientConfig: tlsconfig, } if proxy != nil { transport.Proxy = http.ProxyURL(proxy) } client := &Client{ Host: host, HttpClient: &http.Client{ Transport: transport, }, Username: username, Password: password, UseCookies: useCookies, } if useCookies { options := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } jar, _ := cookiejar.New(&options) client.HttpClient.Jar = jar } return client }
func dialViaHTTPProxy(server string, port uint16) (*tls.Conn, error) { xmppHost := fmt.Sprintf("%s:%d", server, port) fakeRequest := http.Request{ URL: &url.URL{ Scheme: "https", Host: xmppHost, }, } proxyURL, err := http.ProxyFromEnvironment(&fakeRequest) if err != nil { return nil, err } if proxyURL == nil { return nil, nil } dialer := net.Dialer{ KeepAlive: netKeepAlive, Timeout: netTimeout, } conn, err := dialer.Dial("tcp", proxyURL.Host) if err != nil { return nil, fmt.Errorf("Failed to connect to HTTP proxy server: %s", err) } fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", xmppHost, xmppHost) response, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"}) if err != nil { return nil, err } if response.StatusCode != http.StatusOK { return nil, fmt.Errorf("Failed to connect to proxy: %s", response.Status) } return addTLS(server, conn) }
func main() { flag.Usage = usage flag.Parse() args := flag.Args() if len(args) > 0 { args = filter(args) } if ver { fmt.Println("Version:", version) os.Exit(2) } if printV != "A" && printV != "B" { defaultSetting.DumpBody = false } var stdin []byte if runtime.GOOS != "windows" { fi, err := os.Stdin.Stat() if err != nil { panic(err) } if fi.Size() != 0 { stdin, err = ioutil.ReadAll(os.Stdin) if err != nil { log.Fatal("Read from Stdin", err) } } } if *URL == "" { usage() } if strings.HasPrefix(*URL, ":") { urlb := []byte(*URL) if *URL == ":" { *URL = "http://localhost/" } else if len(*URL) > 1 && urlb[1] != '/' { *URL = "http://localhost" + *URL } else { *URL = "http://localhost" + string(urlb[1:]) } } if !strings.HasPrefix(*URL, "http://") && !strings.HasPrefix(*URL, "https://") { *URL = "http://" + *URL } u, err := url.Parse(*URL) if err != nil { log.Fatal(err) } if auth != "" { userpass := strings.Split(auth, ":") if len(userpass) == 2 { u.User = url.UserPassword(userpass[0], userpass[1]) } else { u.User = url.User(auth) } } *URL = u.String() httpreq := getHTTP(*method, *URL, args) if u.User != nil { password, _ := u.User.Password() httpreq.GetRequest().SetBasicAuth(u.User.Username(), password) } // Insecure SSL Support if insecureSSL { httpreq.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) } // Proxy Support if proxy != "" { purl, err := url.Parse(proxy) if err != nil { log.Fatal("Proxy Url parse err", err) } httpreq.SetProxy(http.ProxyURL(purl)) } else { eurl, err := http.ProxyFromEnvironment(httpreq.GetRequest()) if err != nil { log.Fatal("Environment Proxy Url parse err", err) } httpreq.SetProxy(http.ProxyURL(eurl)) } if body != "" { httpreq.Body(body) } if len(stdin) > 0 { var j interface{} d := json.NewDecoder(bytes.NewReader(stdin)) d.UseNumber() err = d.Decode(&j) if err != nil { httpreq.Body(stdin) } else { httpreq.JsonBody(j) } } // AB bench if bench { httpreq.Debug(false) RunBench(httpreq) return } res, err := httpreq.Response() if err != nil { log.Fatalln("can't get the url", err) } // download file if download { var fl string if disposition := res.Header.Get("Content-Disposition"); disposition != "" { fls := strings.Split(disposition, ";") for _, f := range fls { f = strings.TrimSpace(f) if strings.HasPrefix(f, "filename=") { fl = strings.TrimLeft(f, "filename=") } } } if fl == "" { _, fl = filepath.Split(u.Path) } fd, err := os.OpenFile(fl, os.O_RDWR|os.O_CREATE, 0666) if err != nil { log.Fatal("can't create file", err) } if runtime.GOOS != "windows" { fmt.Println(Color(res.Proto, Magenta), Color(res.Status, Green)) for k, v := range res.Header { fmt.Println(Color(k, Gray), ":", Color(strings.Join(v, " "), Cyan)) } } else { fmt.Println(res.Proto, res.Status) for k, v := range res.Header { fmt.Println(k, ":", strings.Join(v, " ")) } } fmt.Println("") contentLength := res.Header.Get("Content-Length") var total int64 if contentLength != "" { total, _ = strconv.ParseInt(contentLength, 10, 64) } fmt.Printf("Downloading to \"%s\"\n", fl) pb := NewProgressBar(total) pb.Start() multiWriter := io.MultiWriter(fd, pb) _, err = io.Copy(multiWriter, res.Body) if err != nil { log.Fatal("Can't Write the body into file", err) } pb.Finish() defer fd.Close() defer res.Body.Close() return } if runtime.GOOS != "windows" { fi, err := os.Stdout.Stat() if err != nil { panic(err) } if fi.Mode()&os.ModeDevice == os.ModeDevice { if printV == "A" || printV == "H" || printV == "B" { dump := httpreq.DumpRequest() if printV == "B" { dps := strings.Split(string(dump), "\n") for i, line := range dps { if len(strings.Trim(line, "\r\n ")) == 0 { dump = []byte(strings.Join(dps[i:], "\n")) break } } } fmt.Println(ColorfulRequest(string(dump))) fmt.Println("") } if printV == "A" || printV == "h" { fmt.Println(Color(res.Proto, Magenta), Color(res.Status, Green)) for k, v := range res.Header { fmt.Println(Color(k, Gray), ":", Color(strings.Join(v, " "), Cyan)) } fmt.Println("") } if printV == "A" || printV == "b" { body := formatResponseBody(res, httpreq, pretty) fmt.Println(ColorfulResponse(body, res.Header.Get("Content-Type"))) } } else { body := formatResponseBody(res, httpreq, pretty) _, err = os.Stdout.WriteString(body) if err != nil { log.Fatal(err) } } } else { if printV == "A" || printV == "H" || printV == "B" { dump := httpreq.DumpRequest() if printV == "B" { dps := strings.Split(string(dump), "\n") for i, line := range dps { if len(strings.Trim(line, "\r\n ")) == 0 { dump = []byte(strings.Join(dps[i:], "\n")) break } } } fmt.Println(string(dump)) fmt.Println("") } if printV == "A" || printV == "h" { fmt.Println(res.Proto, res.Status) for k, v := range res.Header { fmt.Println(k, ":", strings.Join(v, " ")) } fmt.Println("") } if printV == "A" || printV == "b" { body := formatResponseBody(res, httpreq, pretty) fmt.Println(body) } } }
func SmartProxy(req *http.Request) (*url.URL, error) { return http.ProxyFromEnvironment(req) }
func downloadForgeModule(name string, version string, fm ForgeModule, retryCount int) { funcName := funcName() var wgForgeModule sync.WaitGroup extractR, extractW := io.Pipe() saveFileR, saveFileW := io.Pipe() //url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz" fileName := name + "-" + version + ".tar.gz" if !fileExists(config.ForgeCacheDir + name + "-" + version) { baseUrl := config.Forge.Baseurl if len(fm.baseUrl) > 0 { baseUrl = fm.baseUrl } url := baseUrl + "/v3/files/" + fileName req, err := http.NewRequest("GET", url, nil) req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/") req.Header.Set("Connection", "close") proxyURL, err := http.ProxyFromEnvironment(req) if err != nil { Fatalf(funcName + "(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error()) } client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} before := time.Now() Debugf("GETing " + url) resp, err := client.Do(req) duration := time.Since(before).Seconds() Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s") mutex.Lock() syncForgeTime += duration mutex.Unlock() if err != nil { Fatalf(funcName + "(): Error while GETing Forge module " + name + " from " + url + ": " + err.Error()) } defer resp.Body.Close() if resp.Status == "200 OK" { wgForgeModule.Add(1) go func() { defer wgForgeModule.Done() Debugf(funcName + "(): Trying to create " + config.ForgeCacheDir + fileName) out, err := os.Create(config.ForgeCacheDir + fileName) if err != nil { Fatalf(funcName + "(): Error while creating file for Forge module " + config.ForgeCacheDir + fileName + " Error: " + err.Error()) } defer out.Close() io.Copy(out, saveFileR) Debugf(funcName + "(): Finished creating " + config.ForgeCacheDir + fileName) }() wgForgeModule.Add(1) go extractForgeModule(&wgForgeModule, extractR, fileName) wgForgeModule.Add(1) go func() { defer wgForgeModule.Done() // after completing the copy, we need to close // the PipeWriters to propagate the EOF to all // PipeReaders to avoid deadlock defer extractW.Close() defer saveFileW.Close() var mw io.Writer // build the multiwriter for all the pipes mw = io.MultiWriter(extractW, saveFileW) // copy the data into the multiwriter if _, err := io.Copy(mw, resp.Body); err != nil { Fatalf("Error while writing to MultiWriter " + err.Error()) } }() } else { Fatalf(funcName + "(): Unexpected response code while GETing " + url + " " + resp.Status) } } else { Debugf("Using cache for Forge module " + name + " version: " + version) } wgForgeModule.Wait() if checkSum || fm.sha256sum != "" { fm.version = version if doForgeModuleIntegrityCheck(fm) { if retryCount == 0 { Fatalf("downloadForgeModule(): giving up for Puppet module " + name + " version: " + version) } Warnf("Retrying...") purgeDir(config.ForgeCacheDir+fileName, "downloadForgeModule()") purgeDir(strings.Replace(config.ForgeCacheDir+fileName, ".tar.gz", "/", -1), "downloadForgeModule()") // retry if hash sum mismatch found downloadForgeModule(name, version, fm, retryCount-1) } } }
func initMQ(orig_ctx Context, try_proxy bool, proxy string) (ctx Context, err error) { ctx = orig_ctx defer func() { if e := recover(); e != nil { err = fmt.Errorf("initMQ() -> %v", e) } ctx.Channels.Log <- mig.Log{Desc: "leaving initMQ()"}.Debug() }() //Define the AMQP binding ctx.MQ.Bind.Queue = fmt.Sprintf("mig.agt.%s", ctx.Agent.QueueLoc) ctx.MQ.Bind.Key = fmt.Sprintf("mig.agt.%s", ctx.Agent.QueueLoc) // parse the dial string and use TLS if using amqps amqp_uri, err := amqp.ParseURI(AMQPBROKER) if err != nil { panic(err) } ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("AMQP: host=%s, port=%d, vhost=%s", amqp_uri.Host, amqp_uri.Port, amqp_uri.Vhost)}.Debug() if amqp_uri.Scheme == "amqps" { ctx.MQ.UseTLS = true } // create an AMQP configuration with specific timers var dialConfig amqp.Config dialConfig.Heartbeat = 2 * ctx.Sleeper if try_proxy { // if in try_proxy mode, the agent will try to connect to the relay using a CONNECT proxy // but because CONNECT is a HTTP method, not available in AMQP, we need to establish // that connection ourselves, and give it back to the amqp.DialConfig method if proxy == "" { // try to get the proxy from the environemnt (variable HTTP_PROXY) target := "http://" + amqp_uri.Host + ":" + fmt.Sprintf("%d", amqp_uri.Port) req, err := http.NewRequest("GET", target, nil) if err != nil { panic(err) } proxy_url, err := http.ProxyFromEnvironment(req) if err != nil { panic(err) } if proxy_url == nil { panic("Failed to find a suitable proxy in environment") } proxy = proxy_url.Host ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Found proxy at %s", proxy)}.Debug() } ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Connecting via proxy %s", proxy)}.Debug() dialConfig.Dial = func(network, addr string) (conn net.Conn, err error) { // connect to the proxy conn, err = net.DialTimeout("tcp", proxy, 5*time.Second) if err != nil { return } // write a CONNECT request in the tcp connection fmt.Fprintf(conn, "CONNECT "+addr+" HTTP/1.1\r\nHost: "+addr+"\r\n\r\n") // verify status is 200, and flush the buffer status, err := bufio.NewReader(conn).ReadString('\n') if err != nil { return } if status == "" || len(status) < 12 { err = fmt.Errorf("Invalid status received from proxy: '%s'", status[0:len(status)-2]) return } // 9th character in response should be "2" // HTTP/1.0 200 Connection established // ^ if status[9] != '2' { err = fmt.Errorf("Invalid status received from proxy: '%s'", status[0:len(status)-2]) return } ctx.Agent.Env.IsProxied = true ctx.Agent.Env.Proxy = proxy return } } else { dialConfig.Dial = func(network, addr string) (net.Conn, error) { return net.DialTimeout(network, addr, 5*time.Second) } } if ctx.MQ.UseTLS { ctx.Channels.Log <- mig.Log{Desc: "Loading AMQPS TLS parameters"}.Debug() // import the client certificates cert, err := tls.X509KeyPair(AGENTCERT, AGENTKEY) if err != nil { panic(err) } // import the ca cert ca := x509.NewCertPool() if ok := ca.AppendCertsFromPEM(CACERT); !ok { panic("failed to import CA Certificate") } TLSconfig := tls.Config{Certificates: []tls.Certificate{cert}, RootCAs: ca, InsecureSkipVerify: false, Rand: rand.Reader} dialConfig.TLSClientConfig = &TLSconfig } // Open AMQP connection ctx.Channels.Log <- mig.Log{Desc: "Establishing connection to relay"}.Debug() ctx.MQ.conn, err = amqp.DialConfig(AMQPBROKER, dialConfig) if err != nil { ctx.Channels.Log <- mig.Log{Desc: "Connection failed"}.Debug() panic(err) } ctx.MQ.Chan, err = ctx.MQ.conn.Channel() if err != nil { panic(err) } // Limit the number of message the channel will receive at once err = ctx.MQ.Chan.Qos(1, // prefetch count (in # of msg) 0, // prefetch size (in bytes) false) // is global _, err = ctx.MQ.Chan.QueueDeclare(ctx.MQ.Bind.Queue, // Queue name true, // is durable false, // is autoDelete false, // is exclusive false, // is noWait nil) // AMQP args if err != nil { panic(err) } err = ctx.MQ.Chan.QueueBind(ctx.MQ.Bind.Queue, // Queue name ctx.MQ.Bind.Key, // Routing key name mig.Mq_Ex_ToAgents, // Exchange name false, // is noWait nil) // AMQP args if err != nil { panic(err) } // Consume AMQP message into channel ctx.MQ.Bind.Chan, err = ctx.MQ.Chan.Consume(ctx.MQ.Bind.Queue, // queue name "", // some tag false, // is autoAck false, // is exclusive false, // is noLocal false, // is noWait nil) // AMQP args if err != nil { panic(err) } return }
func queryForgeApi(name string, file string) ForgeResult { //url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1) url := "https://forgeapi.puppetlabs.com:443/v3/modules?query=" + name req, err := http.NewRequest("GET", url, nil) if err != nil { log.Fatal("queryForgeApi(): Error creating GET request for Puppetlabs forge API", err) os.Exit(1) } if fileInfo, err := os.Stat(file); err == nil { Debugf("queryForgeApi(): adding If-Modified-Since:" + string(fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + " to Forge query") req.Header.Set("If-Modified-Since", fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) } req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/") req.Header.Set("Connection", "close") proxyUrl, err := http.ProxyFromEnvironment(req) if err != nil { log.Fatal("queryForgeApi(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err) os.Exit(1) } client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} before := time.Now() resp, err := client.Do(req) duration := time.Since(before).Seconds() Verbosef("Querying Forge API " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s") mutex.Lock() syncForgeTime += duration mutex.Unlock() if err != nil { panic(err) } defer resp.Body.Close() if resp.Status == "200 OK" { // need to get latest version body, err := ioutil.ReadAll(resp.Body) //fmt.Println(string(body)) reCurrent := regexp.MustCompile("\\s*\"current_release\": {\n\\s*\"uri\": \"([^\"]+)\",") if m := reCurrent.FindStringSubmatch(string(body)); len(m) > 1 { //fmt.Println(m[1]) if strings.Count(m[1], "-") < 2 { log.Fatal("queryForgeApi(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", m[1], " should contain three '-' characters") os.Exit(1) } else { version := strings.Split(m[1], "-")[2] Debugf("queryForgeApi(): found version " + version + " for " + name + "-latest") mutex.Lock() latestForgeModules[name] = version mutex.Unlock() return ForgeResult{true, version} } } if err != nil { panic(err) } return ForgeResult{false, ""} } else if resp.Status == "304 Not Modified" { Debugf("queryForgeApi(): Got 304 nothing to do for module " + name) return ForgeResult{false, ""} } else { Debugf("queryForgeApi(): Unexpected response code " + resp.Status) return ForgeResult{false, ""} } }
// RestAPICall - general rest method caller func (c *Client) RestAPICall(method Method, path string, options interface{}) ([]byte, error) { log.Debugf("RestAPICall %s - %s%s", method, utils.Sanatize(c.Endpoint), path) var ( Url *url.URL err error req *http.Request ) Url, err = url.Parse(utils.Sanatize(c.Endpoint)) if err != nil { return nil, err } Url.Path += path // Manage the query string c.GetQueryString(Url) log.Debugf("*** url => %s", Url.String()) log.Debugf("*** method => %s", method.String()) // parse url reqUrl, err := url.Parse(Url.String()) if err != nil { return nil, fmt.Errorf("Error with request: %v - %q", Url, err) } // handle options if options != nil { OptionsJSON, err := json.Marshal(options) if err != nil { return nil, err } log.Debugf("*** options => %+v", bytes.NewBuffer(OptionsJSON)) req, err = http.NewRequest(method.String(), reqUrl.String(), bytes.NewBuffer(OptionsJSON)) } else { req, err = http.NewRequest(method.String(), reqUrl.String(), nil) } if err != nil { return nil, fmt.Errorf("Error with request: %v - %q", Url, err) } // setup proxy proxyUrl, err := http.ProxyFromEnvironment(req) if err != nil { return nil, fmt.Errorf("Error with proxy: %v - %q", proxyUrl, err) } if proxyUrl != nil { tr.Proxy = http.ProxyURL(proxyUrl) log.Debugf("*** proxy => %+v", tr.Proxy) } // build the auth headerU for k, v := range c.Option.Headers { log.Debugf("Headers -> %s -> %+v\n", k, v) req.Header.Add(k, v) } // req.SetBasicAuth(c.User, c.APIKey) req.Method = fmt.Sprintf("%s", method.String()) resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() // TODO: CLeanup Later // DEBUGGING WHILE WE WORK // DEBUGGING WHILE WE WORK // fmt.Printf("METHOD --> %+v\n",method) log.Debugf("REQ --> %+v\n", req) log.Debugf("RESP --> %+v\n", resp) log.Debugf("ERROR --> %+v\n", err) // DEBUGGING WHILE WE WORK data, err := ioutil.ReadAll(resp.Body) if !c.isOkStatus(resp.StatusCode) { type apiErr struct { Err string `json:"details"` } var outErr apiErr json.Unmarshal(data, &outErr) return nil, fmt.Errorf("Error in response: %s\n Response Status: %s", outErr.Err, resp.Status) } if err != nil { return nil, err } return data, nil }
func downloadForgeModule(name string, version string) { //url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz" fileName := name + "-" + version + ".tar.gz" if _, err := os.Stat(config.ForgeCacheDir + name + "-" + version); os.IsNotExist(err) { url := "https://forgeapi.puppetlabs.com/v3/files/" + fileName req, err := http.NewRequest("GET", url, nil) req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/") req.Header.Set("Connection", "close") proxyUrl, err := http.ProxyFromEnvironment(req) if err != nil { log.Fatal("downloadForgeModule(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err) os.Exit(1) } client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} before := time.Now() resp, err := client.Do(req) duration := time.Since(before).Seconds() Verbosef("GETing " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s") mutex.Lock() syncForgeTime += duration mutex.Unlock() if err != nil { log.Print("downloadForgeModule(): Error while GETing Forge module ", name, " from ", url, ": ", err) os.Exit(1) } defer resp.Body.Close() if resp.Status == "200 OK" { Debugf("downloadForgeModule(): Trying to create " + config.ForgeCacheDir + fileName) out, err := os.Create(config.ForgeCacheDir + fileName) if err != nil { log.Print("downloadForgeModule(): Error while creating file for Forge module "+config.ForgeCacheDir+fileName, err) os.Exit(1) } defer out.Close() io.Copy(out, resp.Body) file, err := os.Open(config.ForgeCacheDir + fileName) if err != nil { fmt.Println("downloadForgeModule(): Error while opening file", file, err) os.Exit(1) } defer file.Close() var fileReader io.ReadCloser = resp.Body if strings.HasSuffix(fileName, ".gz") { if fileReader, err = pgzip.NewReader(file); err != nil { fmt.Println("downloadForgeModule(): pgzip reader error for module ", fileName, " error:", err) os.Exit(1) } defer fileReader.Close() } tarBallReader := tar.NewReader(fileReader) if err = os.Chdir(config.ForgeCacheDir); err != nil { fmt.Println("downloadForgeModule(): error while chdir to", config.ForgeCacheDir, err) os.Exit(1) } for { header, err := tarBallReader.Next() if err != nil { if err == io.EOF { break } fmt.Println("downloadForgeModule(): error while tar reader.Next() for ", fileName, err) os.Exit(1) } // get the individual filename and extract to the current directory filename := header.Name //Debugf("downloadForgeModule(): Trying to extract file" + filename) switch header.Typeflag { case tar.TypeDir: // handle directory //fmt.Println("Creating directory :", filename) //err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755 if you prefer err = os.MkdirAll(filename, os.FileMode(0755)) // or use 0755 if you prefer if err != nil { fmt.Println("downloadForgeModule(): error while MkdirAll()", filename, err) os.Exit(1) } case tar.TypeReg: // handle normal file //fmt.Println("Untarring :", filename) writer, err := os.Create(filename) if err != nil { fmt.Println("downloadForgeModule(): error while Create()", filename, err) os.Exit(1) } io.Copy(writer, tarBallReader) err = os.Chmod(filename, os.FileMode(0644)) if err != nil { fmt.Println("downloadForgeModule(): error while Chmod()", filename, err) os.Exit(1) } writer.Close() default: fmt.Printf("Unable to untar type : %c in file %s", header.Typeflag, filename) } } } else { log.Print("downloadForgeModule(): Unexpected response code while GETing " + url + resp.Status) os.Exit(1) } } else { Debugf("downloadForgeModule(): Using cache for Forge module " + name + " version: " + version) } }
// websocketConn establishes a connection to the given URL, respecting any proxy configuration in the environment. // A standard proxying setup involves setting the following environment variables // (may be listed in /etc/ecs/ecs.config if using ecs-init): // HTTP_PROXY=http://<your-proxy>/ # HTTPS_PROXY may be set instead or additionally // NO_PROXY=169.254.169.254,/var/run/docker.sock # Directly connect to metadata service and docker socket func (cs *ClientServerImpl) websocketConn(parsedURL *url.URL, request *http.Request) (net.Conn, error) { proxyURL, err := http.ProxyFromEnvironment(request) if err != nil { return nil, err } // url.Host might not have the port, but tls.Dial needs it targetHost := parsedURL.Host if !strings.Contains(targetHost, ":") { targetHost += ":443" } targetHostname, _, err := net.SplitHostPort(targetHost) if err != nil { return nil, err } tlsConfig := tls.Config{ServerName: targetHostname, InsecureSkipVerify: cs.AcceptInvalidCert} timeoutDialer := &net.Dialer{Timeout: wsConnectTimeout} if proxyURL == nil { // directly connect log.Info("Creating poll dialer", "host", parsedURL.Host) return tls.DialWithDialer(timeoutDialer, "tcp", targetHost, &tlsConfig) } // connect via proxy log.Info("Creating poll dialer", "proxy", proxyURL.Host, "host", parsedURL.Host) plainConn, err := timeoutDialer.Dial("tcp", proxyURL.Host) if err != nil { return nil, err } // TLS over an HTTP proxy via CONNECT taken from: https://golang.org/src/net/http/transport.go connectReq := &http.Request{ Method: "CONNECT", URL: &url.URL{Opaque: targetHost}, Host: targetHost, Header: make(http.Header), } if proxyUser := proxyURL.User; proxyUser != nil { username := proxyUser.Username() password, _ := proxyUser.Password() auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) connectReq.Header.Set("Proxy-Authorization", "Basic "+auth) } connectReq.Write(plainConn) // Read response. // Okay to use and discard buffered reader here, because // TLS server will not speak until spoken to. br := bufio.NewReader(plainConn) resp, err := http.ReadResponse(br, connectReq) if err != nil { plainConn.Close() return nil, err } if resp.StatusCode != 200 { plainConn.Close() return nil, errors.New(resp.Status) } tlsConn := tls.Client(plainConn, &tlsConfig) return tlsConn, nil }
func queryForgeAPI(name string, file string, fm ForgeModule) ForgeResult { //url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1) baseUrl := config.Forge.Baseurl if len(fm.baseUrl) > 0 { baseUrl = fm.baseUrl } //url := baseUrl + "/v3/modules?query=" + name url := baseUrl + "/v3/releases?module=" + name + "&owner=" + fm.author + "&sort_by=release_date&limit=1" req, err := http.NewRequest("GET", url, nil) if err != nil { Fatalf("queryForgeAPI(): Error creating GET request for Puppetlabs forge API" + err.Error()) } if fileInfo, err := os.Stat(file); err == nil { Debugf("adding If-Modified-Since:" + string(fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) + " to Forge query") req.Header.Set("If-Modified-Since", fileInfo.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")) } req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/") req.Header.Set("Connection", "close") proxyURL, err := http.ProxyFromEnvironment(req) if err != nil { Fatalf("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error()) } client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} before := time.Now() resp, err := client.Do(req) if err != nil { Fatalf("queryForgeAPI(): Error while issuing the HTTP request to " + url + " Error: " + err.Error()) } duration := time.Since(before).Seconds() Verbosef("Querying Forge API " + url + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s") mutex.Lock() syncForgeTime += duration mutex.Unlock() defer resp.Body.Close() if resp.Status == "200 OK" { // need to get latest version body, err := ioutil.ReadAll(resp.Body) if err != nil { Fatalf("queryForgeAPI(): Error while reading response body for Forge module " + fm.name + " from " + url + ": " + err.Error()) } before := time.Now() currentRelease := gjson.Get(string(body), "results.0").Map() duration := time.Since(before).Seconds() version := currentRelease["version"].String() modulemd5sum := currentRelease["file_md5"].String() moduleFilesize := currentRelease["file_size"].Int() mutex.Lock() forgeJsonParseTime += duration mutex.Unlock() Debugf("found version " + version + " for " + name + "-latest") latestForgeModules.Lock() latestForgeModules.m[name] = version latestForgeModules.Unlock() lastCheckedFile := config.ForgeCacheDir + name + "-latest-last-checked" Debugf("writing last-checked file " + lastCheckedFile) f, _ := os.Create(lastCheckedFile) defer f.Close() return ForgeResult{true, version, modulemd5sum, moduleFilesize} } else if resp.Status == "304 Not Modified" { Debugf("Got 304 nothing to do for module " + name) return ForgeResult{false, "", "", 0} } else { Debugf("Unexpected response code " + resp.Status) return ForgeResult{false, "", "", 0} } }