func (p *Prometheus) gatherURL(url string, acc telegraf.Accumulator) error {
collectDate := time.Now()
var req, err = http.NewRequest("GET", url, nil)
req.Header.Add("Accept", acceptHeader)
var token []byte
var resp *http.Response
tlsCfg, err := internal.GetTLSConfig(
p.SSLCert, p.SSLKey, p.SSLCA, p.InsecureSkipVerify)
if err != nil {
return err
}
var rt http.RoundTripper = &http.Transport{
Dial: (&net.Dialer{
Timeout: 5 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
TLSClientConfig: tlsCfg,
ResponseHeaderTimeout: time.Duration(3 * time.Second),
DisableKeepAlives: true,
}
if p.BearerToken != "" {
token, err = ioutil.ReadFile(p.BearerToken)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+string(token))
}
resp, err = rt.RoundTrip(req)
if err != nil {
return fmt.Errorf("error making HTTP request to %s: %s", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s returned HTTP status %s", url, resp.Status)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading body: %s", err)
}
metrics, err := Parse(body, resp.Header)
if err != nil {
return fmt.Errorf("error reading metrics for %s: %s",
url, err)
}
// Add (or not) collected metrics
for _, metric := range metrics {
tags := metric.Tags()
tags["url"] = url
acc.AddFields(metric.Name(), metric.Fields(), tags, collectDate)
}
return nil
}
func main() {
//Set URL to test --- test will NOT follow redirects
testurl := "https://beacon.walmart.com/vm/ttap.gif?id=10694084&160x600&audience=&creative=1027467&creativetype=rich_media&device=&initiative=209902&placement=SSLTEST&targeting=wmxaudience&vendor=AOD&version=2015802"
// Set log filename
logfile := "testlog.txt"
var DefaultTransport http.RoundTripper = &http.Transport{}
// Set number of requests
for i := 0; i < 500000; i++ {
req, _ := http.NewRequest("GET", testurl, nil)
resp, _ := DefaultTransport.RoundTrip(req)
logger, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatal(err)
}
defer logger.Close()
// Grab specified headers from HTTP response
location := resp.Header.Get("Location")
cookie := resp.Header.Get("Set-Cookie")
t := time.Now().UTC().String()
result := fmt.Sprintf("%s\t%s\t%s\r\n", t, location, cookie)
logger.WriteString(result)
// Wait time before next request
time.Sleep(time.Second)
}
}
// Dial dials to the remote url, and sends streaming request. If it succeeds,
// it returns nil error, and the caller should call Handle function to keep
// receiving appendEntry messages.
func (s *streamClient) start(tr http.RoundTripper, u string, cid types.ID) error {
uu, err := url.Parse(u)
if err != nil {
return fmt.Errorf("parse url %s error: %v", u, err)
}
uu.Path = path.Join(RaftStreamPrefix, s.id.String())
req, err := http.NewRequest("GET", uu.String(), nil)
if err != nil {
return fmt.Errorf("new request to %s error: %v", u, err)
}
req.Header.Set("X-Etcd-Cluster-ID", cid.String())
req.Header.Set("X-Raft-To", s.to.String())
req.Header.Set("X-Raft-Term", strconv.FormatUint(s.term, 10))
resp, err := tr.RoundTrip(req)
if err != nil {
return fmt.Errorf("error posting to %q: %v", u, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return fmt.Errorf("unhandled http status %d", resp.StatusCode)
}
s.closer = resp.Body
go s.handle(resp.Body)
log.Printf("rafthttp: starting client stream to %s at term %d", s.to, s.term)
return nil
}
// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
var transport http.RoundTripper
if t.Transport != nil {
transport = t.Transport
} else {
transport = http.DefaultTransport
}
// To set extra querystring params, we must make a copy of the Request so
// that we don't modify the Request we were given. This is required by the
// specification of http.RoundTripper.
req = cloneRequest(req)
child := t.Recorder.Child()
if t.SetName {
child.Name(req.URL.Host)
}
SetSpanIDHeader(req.Header, child.SpanID)
e := NewClientEvent(req)
e.ClientSend = time.Now()
// Make the HTTP request.
resp, err := transport.RoundTrip(req)
e.ClientRecv = time.Now()
if err == nil {
e.Response = responseInfo(resp)
} else {
e.Response.StatusCode = -1
}
child.Event(e)
return resp, err
}
func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
var u http.RoundTripper
if t.Transport != nil {
u = t.Transport
} else {
u = http.DefaultTransport
}
reqBytes, err := httputil.DumpRequestOut(req, true)
if err != nil {
return nil, err
}
t.Writer.Write(reqBytes)
resp, err := u.RoundTrip(req)
if err != nil {
return nil, err
}
respBytes, err := httputil.DumpResponse(resp, true)
if err != nil {
return nil, err
}
t.Writer.Write(respBytes)
return resp, nil
}
// newStreamClient starts and returns a new started stream client.
// The caller should call stop when finished, to shut it down.
func newStreamReader(id, to, cid types.ID, term uint64, tr http.RoundTripper, u string, r Raft) (*streamReader, error) {
s := &streamReader{
id: id,
to: to,
term: term,
r: r,
done: make(chan struct{}),
}
uu, err := url.Parse(u)
if err != nil {
return nil, fmt.Errorf("parse url %s error: %v", u, err)
}
uu.Path = path.Join(RaftStreamPrefix, s.id.String())
req, err := http.NewRequest("GET", uu.String(), nil)
if err != nil {
return nil, fmt.Errorf("new request to %s error: %v", u, err)
}
req.Header.Set("X-Etcd-Cluster-ID", cid.String())
req.Header.Set("X-Raft-To", s.to.String())
req.Header.Set("X-Raft-Term", strconv.FormatUint(s.term, 10))
resp, err := tr.RoundTrip(req)
if err != nil {
return nil, fmt.Errorf("error posting to %q: %v", u, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
}
s.closer = resp.Body
go s.handle(resp.Body)
log.Printf("rafthttp: starting client stream to %s at term %d", s.to, s.term)
return s, nil
}
func roundTrip(rt http.RoundTripper, req *http.Request) (body []byte, err error) {
resp, err := rt.RoundTrip(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}
// ByteThrottledRoundTripper wraps another RoundTripper rt,
// throttling all requests to the specified byte rate.
func ByteThrottledRoundTripper(rt http.RoundTripper, rate int64) http.RoundTripper {
freq := time.Duration(1 * time.Millisecond)
bucket := tb.NewBucket(rate, freq)
return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
got := bucket.Take(r.ContentLength)
for got < r.ContentLength {
got += bucket.Take(r.ContentLength - got)
time.Sleep(freq)
}
return rt.RoundTrip(r)
})
}
// ReqThrottledRoundTripper wraps another RoundTripper rt,
// throttling all requests to the specified request rate.
func ReqThrottledRoundTripper(rt http.RoundTripper, rate int64) http.RoundTripper {
freq := time.Duration(1e9 / rate)
bucket := tb.NewBucket(rate, freq)
return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
got := bucket.Take(1)
for got != 1 {
got = bucket.Take(1)
time.Sleep(freq)
}
return rt.RoundTrip(r)
})
}
func request(rt http.RoundTripper, requestURL string, requestHeaders http.Header) (*http.Response, error) {
// Build the request
req, err := http.NewRequest("GET", requestURL, nil)
if err != nil {
return nil, err
}
for k, v := range requestHeaders {
req.Header[k] = v
}
req.Header.Set(CSRFTokenHeader, "1")
// Make the request
return rt.RoundTrip(req)
}
// HTTPGet creates a new http request
func HTTPGet(url string, follow, insecure bool, h map[string]string, timeout ...int) (*http.Response, error) {
// timeout in seconds defaults to 5
var t int = 5
if len(timeout) > 0 {
t = timeout[0]
}
// if insecure = true, skip ssl verification
tr := &http.Transport{
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure},
ResponseHeaderTimeout: time.Duration(t) * time.Second,
}
client := &http.Client{}
client.Transport = tr
// create a new request
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "epazote")
// set custom headers on request
if h != nil {
for k, v := range h {
req.Header.Set(k, v)
}
}
if follow {
res, err := client.Do(req)
if err != nil {
return nil, err
}
return res, nil
}
// not follow redirects
var DefaultTransport http.RoundTripper = tr
res, err := DefaultTransport.RoundTrip(req)
if err != nil {
return nil, err
}
return res, nil
}
func (S) TestTransport(c *gocheck.C) {
var t http.RoundTripper = &Transport{
Message: "Ok",
Status: http.StatusOK,
Headers: map[string][]string{"Authorization": {"something"}},
}
req, _ := http.NewRequest("GET", "/", nil)
r, err := t.RoundTrip(req)
c.Assert(err, gocheck.IsNil)
c.Assert(r.StatusCode, gocheck.Equals, http.StatusOK)
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
c.Assert(string(b), gocheck.Equals, "Ok")
c.Assert(r.Header.Get("Authorization"), gocheck.Equals, "something")
}
// NewRoundTripper creates an http.RoundTripper to instrument external requests.
// This RoundTripper must be used in same the goroutine as the other uses of the
// Transaction's SegmentTracer methods. http.DefaultTransport is used if an
// http.RoundTripper is not provided.
//
// client := &http.Client{}
// client.Transport = newrelic.NewRoundTripper(txn, nil)
// resp, err := client.Get("http://example.com/")
//
func NewRoundTripper(txn Transaction, original http.RoundTripper) http.RoundTripper {
return roundTripperFunc(func(request *http.Request) (*http.Response, error) {
segment := StartExternalSegment(txn, request)
if nil == original {
original = http.DefaultTransport
}
response, err := original.RoundTrip(request)
segment.Response = response
segment.End()
return response, err
})
}
// NewHTTPPoolTransport returns a function that must be used in groupcache.HTTPPool.Transport.
//
// rt is optional, http.DefaultTransport is used by default.
func NewHTTPPoolTransport(rt http.RoundTripper) func(groupcache.Context) http.RoundTripper {
if rt == nil {
rt = http.DefaultTransport
}
return func(ctx groupcache.Context) http.RoundTripper {
return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
if ctx, ok := ctx.(*Context); ok && ctx != nil {
err := setContext(req, ctx)
if err != nil {
return nil, err
}
}
return rt.RoundTrip(req)
})
}
}
// RoundTrip implements the http.RoundTripper interface.
//
// For higher-level HTTP client support (such as handling of cookies
// and redirects), see Get, Post, and the Client type.
func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
if req.URL == nil {
closeBody(req)
return nil, errors.New("http: nil Request.URL")
}
if req.Header == nil {
closeBody(req)
return nil, errors.New("http: nil Request.Header")
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
t.altMu.RLock()
var rt http.RoundTripper
if t.altProto != nil {
rt = t.altProto[req.URL.Scheme]
}
t.altMu.RUnlock()
if rt == nil {
closeBody(req)
return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
}
return rt.RoundTrip(req)
}
if req.URL.Host == "" {
closeBody(req)
return nil, errors.New("http: no Host in request URL")
}
treq := &transportRequest{Request: req}
cm, err := t.connectMethodForRequest(treq)
if err != nil {
closeBody(req)
return nil, err
}
// Get the cached or newly-created connection to either the
// host (for http or https), the http proxy, or the http proxy
// pre-CONNECTed to https server. In any case, we'll be ready
// to send it requests.
pconn, err := t.getConn(req, cm)
if err != nil {
t.setReqCanceler(req, nil)
closeBody(req)
return nil, err
}
return pconn.roundTrip(treq)
}
// RoundTrip implements the RoundTripper interface.
func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
var transport http.RoundTripper
if t.Transport != nil {
transport = t.Transport
} else {
transport = http.DefaultTransport
}
// To set extra querystring params, we must make a copy of the Request so
// that we don't modify the Request we were given. This is required by the
// specification of http.RoundTripper.
req = cloneRequest(req)
req.SetBasicAuth(t.Username, t.Password)
// Make the HTTP request.
return transport.RoundTrip(req)
}
func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
var transport http.RoundTripper
if t.Transport != nil {
transport = t.Transport
} else {
transport = http.DefaultTransport
}
req2 := SetHeaders(req, t.Span)
event := NewEvent(t.Span, ClientRequestSent)
t.Collector.Record(event)
resp, err := transport.RoundTrip(req2)
event = NewEvent(t.Span, ClientRequestReceived)
t.Collector.Record(event)
return resp, err
}
func downloadFile(d *drive.Service, t http.RoundTripper, f *drive.File) {
downloadUrl := f.DownloadUrl
if downloadUrl == "" {
log.Fatal("An error occurred: File is not downloadable")
}
req, _ := http.NewRequest("GET", downloadUrl, nil)
resp, _ := t.RoundTrip(req)
defer resp.Body.Close()
file, _ := os.Create(eventFilePath)
defer file.Close()
if _, err := io.Copy(file, resp.Body); err != nil {
log.Fatal(err)
}
}
// Accepts a Httpaction and a one-way channel to write the results to.
func DoHttpRequest(httpAction HttpAction, resultsChannel chan HttpReqResult, sessionMap map[string]string) {
req := buildHttpRequest(httpAction, sessionMap)
start := time.Now()
var DefaultTransport http.RoundTripper = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
resp, err := DefaultTransport.RoundTrip(req)
if err != nil {
log.Printf("HTTP request failed: %s", err)
} else {
elapsed := time.Since(start)
responseBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
//log.Fatal(err)
log.Printf("Reading HTTP response failed: %s\n", err)
httpReqResult := buildHttpResult(0, resp.StatusCode, elapsed.Nanoseconds(), httpAction.Title)
resultsChannel <- httpReqResult
} else {
defer resp.Body.Close()
if httpAction.StoreCookie != "" {
for _, cookie := range resp.Cookies() {
if cookie.Name == httpAction.StoreCookie {
sessionMap["____"+cookie.Name] = cookie.Value
}
}
}
// if action specifies response action, parse using regexp/jsonpath
processResult(httpAction, sessionMap, responseBody)
httpReqResult := buildHttpResult(len(responseBody), resp.StatusCode, elapsed.Nanoseconds(), httpAction.Title)
resultsChannel <- httpReqResult
}
}
}
// RoundTrip implements http.RoundTripper.
func (t *TicketAuthedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
var transport http.RoundTripper
if t.Transport != nil {
transport = t.Transport
} else {
transport = http.DefaultTransport
}
if len(t.SignedTicketStrings) > 0 {
// To set extra headers, we must make a copy of the Request so
// that we don't modify the Request we were given. This is
// required by the specification of http.RoundTripper.
req = cloneRequest(req)
for _, tstr := range t.SignedTicketStrings {
req.Header.Add("authorization", TicketAuthScheme+tstr)
}
}
// Make the HTTP request.
return transport.RoundTrip(req)
}
func (S) TestConditionalTransport(c *gocheck.C) {
var t http.RoundTripper = &ConditionalTransport{
Transport: Transport{
Message: "Ok",
Status: http.StatusOK,
},
CondFunc: func(req *http.Request) bool {
return req.URL.Path == "/something"
},
}
req, _ := http.NewRequest("GET", "/something", nil)
r, err := t.RoundTrip(req)
c.Assert(err, gocheck.IsNil)
c.Assert(r.StatusCode, gocheck.Equals, http.StatusOK)
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
c.Assert(string(b), gocheck.Equals, "Ok")
req, _ = http.NewRequest("GET", "/", nil)
r, err = t.RoundTrip(req)
c.Assert(err, gocheck.NotNil)
c.Assert(err.Error(), gocheck.Equals, "condition failed")
c.Assert(r.StatusCode, gocheck.Equals, http.StatusInternalServerError)
}
func benchmarkEchoProtoHTTP(b *testing.B, size int, accept func(net.Listener, *tls.Config) error, roundTripper http.RoundTripper) {
var url string
benchmarkEcho(b, size, accept,
func(addr net.Addr) {
url = fmt.Sprintf("https://%s", addr)
},
nil,
func(echoMsg string) string {
args := EchoRequest{Msg: echoMsg}
reqBody, err := proto.Marshal(&args)
if err != nil {
b.Fatal(err)
}
req, err := http.NewRequest("POST", url, bytes.NewReader(reqBody))
if err != nil {
b.Fatal(err)
}
req.Header.Set("Content-Type", xProtobuf)
resp, err := roundTripper.RoundTrip(req)
if err != nil {
b.Fatal(err)
}
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
b.Fatal(err)
}
if err := resp.Body.Close(); err != nil {
b.Fatal(err)
}
reply := EchoResponse{}
if err := proto.Unmarshal(respBody, &reply); err != nil {
b.Fatal(err)
}
return reply.Msg
},
)
}
func (h ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
activeConnections.Add(1)
defer activeConnections.Done()
conf := GetConfig()
if !conf.ACLsLoaded {
http.Error(w, "Redwood proxy configuration needs to be updated for this version of Redwood.\n(Use ACLs)", 500)
return
}
if len(r.URL.String()) > 10000 {
http.Error(w, "URL too long", http.StatusRequestURITooLong)
return
}
client := r.RemoteAddr
host, _, err := net.SplitHostPort(client)
if err == nil {
client = host
}
if conf.AuthCacheTime > 0 {
auth := r.Header.Get("Proxy-Authorization")
if auth == "" {
authCacheLock.RLock()
ar, ok := authCache[client]
authCacheLock.RUnlock()
if ok && time.Now().Sub(ar.Time) < time.Duration(conf.AuthCacheTime)*time.Second {
r.Header.Set("Proxy-Authorization", ar.ProxyAuthorization)
}
} else {
authCacheLock.Lock()
authCache[client] = authRecord{
ProxyAuthorization: auth,
Time: time.Now(),
}
authCacheLock.Unlock()
}
}
if r.Header.Get("Proxy-Authorization") != "" {
user, pass := ProxyCredentials(r)
if !conf.ValidCredentials(user, pass) {
log.Printf("Incorrect username or password from %v: %q:%q", r.RemoteAddr, user, pass)
r.Header.Del("Proxy-Authorization")
}
}
// Reconstruct the URL if it is incomplete (i.e. on a transparent proxy).
if r.URL.Host == "" {
r.URL.Host = r.Host
}
if r.URL.Scheme == "" {
if h.TLS {
r.URL.Scheme = "https"
} else {
r.URL.Scheme = "http"
}
}
var userAgent string
if conf.LogUserAgent {
userAgent = r.Header.Get("User-Agent")
}
if realHost, ok := conf.VirtualHosts[r.Host]; ok {
r.Host = realHost
r.URL.Host = realHost
}
user := client
var authUser string
if h.user != "" {
authUser = h.user
} else if u, _ := ProxyCredentials(r); u != "" {
authUser = u
}
if authUser != "" {
user = authUser
}
tally := conf.URLRules.MatchingRules(r.URL)
scores := conf.categoryScores(tally)
categories := conf.significantCategories(scores)
reqACLs := conf.ACLs.requestACLs(r, authUser)
possibleActions := []string{
"allow",
"block",
"block-invisible",
}
if r.Header.Get("Proxy-Authorization") == "" && !h.TLS {
possibleActions = append(possibleActions, "require-auth")
}
if r.Method == "CONNECT" && conf.TLSReady {
possibleActions = append(possibleActions, "ssl-bump")
}
thisRule, ignored := conf.ChooseACLCategoryAction(reqACLs, categories, possibleActions...)
if r.Method == "CONNECT" && conf.TLSReady && thisRule.Action == "" {
// If the result is unclear, go ahead and start to bump the connection.
// The ACLs will be checked one more time anyway.
thisRule.Action = "ssl-bump"
}
switch thisRule.Action {
case "require-auth":
conf.send407(w)
log.Printf("Missing required proxy authentication from %v to %v", r.RemoteAddr, r.URL)
return
case "block":
conf.showBlockPage(w, r, user, tally, scores, thisRule)
logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
return
case "block-invisible":
showInvisibleBlock(w)
logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
return
case "ssl-bump":
conn, err := newHijackedConn(w)
if err != nil {
fmt.Fprintln(conn, "HTTP/1.1 500 Internal Server Error")
fmt.Fprintln(conn)
fmt.Fprintln(conn, err)
conn.Close()
return
}
fmt.Fprint(conn, "HTTP/1.1 200 Connection Established\r\n\r\n")
SSLBump(conn, r.URL.Host, user, authUser)
return
}
if r.Host == localServer {
conf.ServeMux.ServeHTTP(w, r)
return
}
if r.Method == "CONNECT" {
conn, err := newHijackedConn(w)
if err != nil {
fmt.Fprintln(conn, "HTTP/1.1 500 Internal Server Error")
fmt.Fprintln(conn)
fmt.Fprintln(conn, err)
conn.Close()
return
}
fmt.Fprint(conn, "HTTP/1.1 200 Connection Established\r\n\r\n")
logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
connectDirect(conn, r.URL.Host, nil)
return
}
if r.Header.Get("Upgrade") == "websocket" {
h.makeWebsocketConnection(w, r)
return
}
r.Header.Add("Via", r.Proto+" Redwood")
r.Header.Add("X-Forwarded-For", client)
gzipOK := !conf.DisableGZIP && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && !lanAddress(client)
r.Header.Del("Accept-Encoding")
urlChanged := conf.changeQuery(r.URL)
if !urlChanged {
// Rebuild the URL in a way that will preserve which characters are escaped
// and which aren't, for compatibility with broken servers.
rawURL := r.RequestURI
if strings.HasPrefix(rawURL, r.URL.Scheme) {
rawURL = rawURL[len(r.URL.Scheme):]
rawURL = strings.TrimPrefix(rawURL, "://")
slash := strings.Index(rawURL, "/")
if slash == -1 {
rawURL = "/"
} else {
rawURL = rawURL[slash:]
}
}
q := strings.Index(rawURL, "?")
if q != -1 {
rawURL = rawURL[:q]
}
if strings.HasPrefix(rawURL, "//") {
// The path should start with a single slash not two.
rawURL = rawURL[1:]
}
r.URL.Opaque = rawURL
}
proxied := false
var rt http.RoundTripper
if h.rt == nil {
if r.URL.Opaque != "" && transport.Proxy != nil {
if p, _ := transport.Proxy(r); p != nil {
// If the request is going through a proxy, the host needs to be
// included in the opaque element.
r.URL.Opaque = "//" + r.URL.Host + r.URL.Opaque
proxied = true
}
}
rt = &transport
} else {
rt = h.rt
}
if !proxied {
r.Header.Del("Proxy-Authorization")
}
resp, err := rt.RoundTrip(r)
r.URL.Opaque = ""
if err != nil {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
log.Printf("error fetching %s: %s", r.URL, err)
logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
return
}
defer resp.Body.Close()
// Prevent switching to QUIC.
resp.Header.Del("Alternate-Protocol")
originalContentType := resp.Header.Get("Content-Type")
fixContentType(resp)
respACLs := conf.ACLs.responseACLs(resp)
acls := unionACLSets(reqACLs, respACLs)
thisRule, ignored = conf.ChooseACLCategoryAction(acls, categories, "allow", "block", "block-invisible", "hash-image", "phrase-scan")
if thisRule.Action == "" {
thisRule.Action = "allow"
}
switch thisRule.Action {
case "allow":
resp.Header.Set("Content-Type", originalContentType)
copyResponseHeader(w, resp)
n, err := io.Copy(w, resp.Body)
if err != nil {
log.Printf("error while copying response (URL: %s): %s", r.URL, err)
}
logAccess(r, resp, int(n), false, user, tally, scores, thisRule, "", ignored, userAgent)
return
case "block":
conf.showBlockPage(w, r, user, tally, scores, thisRule)
logAccess(r, resp, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
return
case "block-invisible":
showInvisibleBlock(w)
logAccess(r, resp, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
return
}
lr := &io.LimitedReader{
R: resp.Body,
N: 1e6,
}
content, err := ioutil.ReadAll(lr)
if err != nil {
log.Printf("error while reading response body (URL: %s): %s", r.URL, err)
}
if lr.N == 0 {
log.Println("response body too long to filter:", r.URL)
resp.Header.Set("Content-Type", originalContentType)
var dest io.Writer = w
if gzipOK {
resp.Header.Set("Content-Encoding", "gzip")
resp.Header.Del("Content-Length")
gzw := gzip.NewWriter(w)
defer gzw.Close()
dest = gzw
}
copyResponseHeader(w, resp)
dest.Write(content)
n, err := io.Copy(dest, resp.Body)
if err != nil {
log.Printf("error while copying response (URL: %s): %s", r.URL, err)
}
logAccess(r, resp, int(n)+len(content), false, user, tally, scores, ACLActionRule{Action: "allow", Needed: []string{"too-long-to-filter"}}, "", ignored, userAgent)
return
}
modified := false
pageTitle := ""
switch thisRule.Action {
case "phrase-scan":
contentType := resp.Header.Get("Content-Type")
_, cs, _ := charset.DetermineEncoding(content, contentType)
if strings.Contains(contentType, "html") {
var doc *html.Node
if conf.LogTitle {
doc, err = parseHTML(content, cs)
if err != nil {
log.Printf("Error parsing HTML from %s: %s", r.URL, err)
} else {
t := titleSelector.MatchFirst(doc)
if t != nil {
if titleText := t.FirstChild; titleText != nil && titleText.Type == html.TextNode {
pageTitle = titleText.Data
}
}
}
}
modified = conf.pruneContent(r.URL, &content, cs, acls, doc)
if modified {
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
cs = "utf-8"
resp.Header.Del("Content-Length")
}
}
conf.scanContent(content, contentType, cs, tally)
case "hash-image":
img, _, err := image.Decode(bytes.NewReader(content))
if err != nil {
log.Printf("Error decoding image from %v: %v", r.URL, err)
break
}
hash := dhash.New(img)
for _, h := range conf.ImageHashes {
if dhash.Distance(hash, h) <= conf.DhashThreshold {
tally[rule{imageHash, h.String()}]++
}
}
}
scores = conf.categoryScores(tally)
categories = conf.significantCategories(scores)
thisRule, ignored = conf.ChooseACLCategoryAction(acls, categories, "allow", "block", "block-invisible")
if thisRule.Action == "" {
thisRule.Action = "allow"
}
switch thisRule.Action {
case "block":
conf.showBlockPage(w, r, user, tally, scores, thisRule)
logAccess(r, resp, len(content), modified, user, tally, scores, thisRule, pageTitle, ignored, userAgent)
return
case "block-invisible":
showInvisibleBlock(w)
logAccess(r, resp, len(content), modified, user, tally, scores, thisRule, pageTitle, ignored, userAgent)
return
}
if !modified {
resp.Header.Set("Content-Type", originalContentType)
}
if gzipOK && len(content) > 1000 {
resp.Header.Set("Content-Encoding", "gzip")
resp.Header.Del("Content-Length")
copyResponseHeader(w, resp)
gzw := gzip.NewWriter(w)
gzw.Write(content)
gzw.Close()
} else {
copyResponseHeader(w, resp)
w.Write(content)
}
logAccess(r, resp, len(content), modified, user, tally, scores, thisRule, pageTitle, ignored, userAgent)
}
var after round_tripper.AfterRoundTrip
servingBackend := true
proxyRoundTripper = round_tripper.NewProxyRoundTripper(
servingBackend, transport, endpointIterator, logger, after)
})
Context("when backend is unavailable", func() {
BeforeEach(func() {
transport.RoundTripStub = func(req *http.Request) (*http.Response, error) {
return nil, dialError
}
})
It("retries 3 times", func() {
resp.HeaderReturns(make(http.Header))
_, err := proxyRoundTripper.RoundTrip(req)
Expect(err).To(HaveOccurred())
Expect(endpointIterator.NextCallCount()).To(Equal(3))
})
})
Context("when there are no more endpoints available", func() {
BeforeEach(func() {
endpointIterator.NextReturns(nil)
})
It("returns a 502 BadGateway error", func() {
resp.HeaderReturns(make(http.Header))
backendRes, err := proxyRoundTripper.RoundTrip(req)
Expect(err).To(HaveOccurred())
Expect(backendRes).To(BeNil())
// GetRepository yields a repository matching the given name, if any exists.
// Repository may be of various forms, in which case omitted elements take
// assumed defaults.
//
// helloworld -> index.docker.io/library/helloworld
// foo/helloworld -> index.docker.io/foo/helloworld
// quay.io/foo/helloworld -> quay.io/foo/helloworld
//
func (c *Client) GetRepository(repository string) ([]flux.ImageDescription, error) {
var host, org, image string
parts := strings.Split(repository, "/")
switch len(parts) {
case 1:
host = dockerHubHost
org = dockerHubLibrary
image = parts[0]
case 2:
host = dockerHubHost
org = parts[0]
image = parts[1]
case 3:
host = parts[0]
org = parts[1]
image = parts[2]
default:
return nil, fmt.Errorf(`expected image name as either "<host>/<org>/<image>", "<org>/<image>", or "<image>"`)
}
hostlessImageName := fmt.Sprintf("%s/%s", org, image)
httphost := "https://" + host
// quay.io wants us to use cookies for authorisation, so we have
// to construct one (the default client has none). This means a
// bit more constructing things to be able to make a registry
// client literal, rather than calling .New()
jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
if err != nil {
return nil, err
}
auth := c.Credentials.credsFor(host)
// A context we'll use to cancel requests on error
ctx, cancel := context.WithCancel(context.Background())
// Use the wrapper to fix headers for quay.io, and remember bearer tokens
var transport http.RoundTripper = &wwwAuthenticateFixer{transport: http.DefaultTransport}
// Now the auth-handling wrappers that come with the library
transport = dockerregistry.WrapTransport(transport, httphost, auth.username, auth.password)
client := &dockerregistry.Registry{
URL: httphost,
Client: &http.Client{
Transport: roundtripperFunc(func(r *http.Request) (*http.Response, error) {
return transport.RoundTrip(r.WithContext(ctx))
}),
Jar: jar,
},
Logf: dockerregistry.Quiet,
}
tags, err := client.Tags(hostlessImageName)
if err != nil {
cancel()
return nil, err
}
// the hostlessImageName is canonicalised, in the sense that it
// includes "library" as the org, if unqualified -- e.g.,
// `library/nats`. We need that to fetch the tags etc. However, we
// want the results to use the *actual* name of the images to be
// as supplied, e.g., `nats`.
return c.tagsToRepository(cancel, client, hostlessImageName, repository, tags)
}
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
if !f.SiteMatcher.Match(req.Host) {
return ctx, nil, nil
}
var tr http.RoundTripper = f.GAETransport
if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
u := strings.Replace(req.URL.String(), "http://", "https://", 1)
glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
resp := &http.Response{
StatusCode: http.StatusMovedPermanently,
Header: http.Header{
"Location": []string{u},
},
Request: req,
Close: true,
ContentLength: -1,
}
glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if f.DirectSiteMatcher.Match(req.Host) {
if req.URL.Path == "/url" {
if rawurl := req.URL.Query().Get("url"); rawurl != "" {
if u, err := url.Parse(rawurl); err == nil {
if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
rawurl = strings.Replace(rawurl, "http://", "https://", 1)
}
}
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
}
if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
tr = f.DirectTransport
if s := req.Header.Get("Connection"); s != "" {
if s1 := strings.ToLower(s); s != s1 {
req.Header.Set("Connection", s1)
}
}
}
}
if tr != f.DirectTransport && req.Method == http.MethodOptions {
if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: false,
ContentLength: -1,
}
for _, s := range v.([]string) {
parts := strings.SplitN(s, ":", 2)
if len(parts) == 2 {
resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
}
}
if origin := req.Header.Get("Origin"); origin != "" {
resp.Header.Set("Access-Control-Allow-Origin", origin)
}
if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
resp.Header.Set("Access-Control-Allow-Headers", headers)
}
glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
prefix := "FETCH"
if tr == f.DirectTransport {
prefix = "DIRECT"
}
resp, err := tr.RoundTrip(req)
if err != nil {
glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
if tr == f.DirectTransport {
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
// f.MultiDialer.ClearCache()
helpers.TryCloseConnections(tr)
}
}
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return ctx, nil, err
}
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/") &&
resp.Header.Get("Content-Encoding") == "" &&
f.ForceDeflateMatcher.Match(req.Host) {
buf := make([]byte, 1024)
n, err := resp.Body.Read(buf)
if err != nil {
defer resp.Body.Close()
return ctx, nil, err
}
buf = buf[:n]
switch {
case helpers.IsGzip(buf):
resp.Header.Set("Content-Encoding", "gzip")
case helpers.IsBinary(buf):
resp.Header.Set("Content-Encoding", "deflate")
}
resp.Body = helpers.NewMultiReadCloser(bytes.NewReader(buf), resp.Body)
}
if resp != nil && resp.Header != nil {
resp.Header.Del("Alt-Svc")
resp.Header.Del("Alternate-Protocol")
}
glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, err
}
func NewFilter(config *Config) (filters.Filter, error) {
dnsServers := make([]net.IP, 0)
for _, s := range config.DNSServers {
if ip := net.ParseIP(s); ip != nil {
dnsServers = append(dnsServers, ip)
}
}
GoogleG2PKP, err := base64.StdEncoding.DecodeString(config.GoogleG2PKP)
if err != nil {
return nil, err
}
googleTLSConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
ServerName: "www.microsoft.com",
ClientSessionCache: tls.NewLRUClientSessionCache(config.TLSConfig.ClientSessionCacheSize),
CipherSuites: []uint16{
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
},
}
switch config.TLSConfig.Version {
case "TLSv12", "TLSv1.2":
googleTLSConfig.MinVersion = tls.VersionTLS12
case "TLSv11", "TLSv1.1":
googleTLSConfig.MinVersion = tls.VersionTLS11
default:
googleTLSConfig.MinVersion = tls.VersionTLS10
}
pickupCiphers := func(names []string) []uint16 {
ciphers := make([]uint16, 0)
for _, name := range names {
cipher := helpers.Cipher(name)
if cipher == 0 {
glog.Fatalf("GAE: cipher %#v is not supported.", name)
}
ciphers = append(ciphers, cipher)
}
helpers.ShuffleUint16s(ciphers)
ciphers = ciphers[:1+rand.Intn(len(ciphers))]
ciphers1 := []uint16{}
for _, name := range []string{
"TLS_RSA_WITH_AES_256_CBC_SHA256",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
} {
if !helpers.ContainsString(names, name) {
if c := helpers.Cipher(name); c != 0 {
ciphers1 = append(ciphers1, c)
}
}
}
helpers.ShuffleUint16s(ciphers1)
ciphers1 = ciphers1[:rand.Intn(len(ciphers1))]
ciphers = append(ciphers, ciphers1...)
helpers.ShuffleUint16s(ciphers)
return ciphers
}
googleTLSConfig.CipherSuites = pickupCiphers(config.TLSConfig.Ciphers)
if len(config.TLSConfig.ServerName) > 0 {
googleTLSConfig.ServerName = config.TLSConfig.ServerName[rand.Intn(len(config.TLSConfig.ServerName))]
}
if !config.DisableHTTP2 {
googleTLSConfig.NextProtos = []string{"h2", "h2-14", "http/1.1"}
}
if config.Site2Alias == nil {
config.Site2Alias = make(map[string]string)
}
for key, value := range config.SiteToAlias {
config.Site2Alias[key] = value
}
config.SiteToAlias = config.Site2Alias
hostmap := map[string][]string{}
for key, value := range config.HostMap {
hostmap[key] = helpers.UniqueStrings(value)
}
d := &net.Dialer{
KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
Timeout: time.Duration(config.Transport.Dialer.Timeout) * time.Second,
DualStack: config.Transport.Dialer.DualStack,
}
md := &dialer.MultiDialer{
Dialer: d,
DisableIPv6: config.DisableIPv6,
ForceIPv6: config.ForceIPv6,
SSLVerify: config.SSLVerify,
EnableRemoteDNS: config.EnableRemoteDNS,
LogToStderr: flag.Lookup("logtostderr") != nil,
TLSConfig: nil,
SiteToAlias: helpers.NewHostMatcherWithString(config.SiteToAlias),
IPBlackList: lrucache.NewLRUCache(1024),
HostMap: hostmap,
GoogleTLSConfig: googleTLSConfig,
GoogleG2PKP: GoogleG2PKP,
DNSServers: dnsServers,
DNSCache: lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
DNSCacheExpiry: time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
TLSConnDuration: lrucache.NewLRUCache(8192),
TLSConnError: lrucache.NewLRUCache(8192),
TLSConnReadBuffer: config.Transport.Dialer.SocketReadBuffer,
ConnExpiry: 5 * time.Minute,
Level: config.Transport.Dialer.Level,
}
for _, ip := range config.IPBlackList {
md.IPBlackList.Set(ip, struct{}{}, time.Time{})
}
var tr http.RoundTripper
t1 := &http.Transport{
Dial: md.Dial,
DialTLS: md.DialTLS,
DisableKeepAlives: config.Transport.DisableKeepAlives,
DisableCompression: config.Transport.DisableCompression,
ResponseHeaderTimeout: time.Duration(config.Transport.ResponseHeaderTimeout) * time.Second,
IdleConnTimeout: time.Duration(config.Transport.IdleConnTimeout) * time.Second,
MaxIdleConnsPerHost: config.Transport.MaxIdleConnsPerHost,
}
if config.Transport.Proxy.Enabled {
fixedURL, err := url.Parse(config.Transport.Proxy.URL)
if err != nil {
glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
}
dialer, err := proxy.FromURL(fixedURL, d, &dialer.MultiResolver{md})
if err != nil {
glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
}
t1.Dial = dialer.Dial
t1.DialTLS = nil
t1.Proxy = nil
t1.TLSClientConfig = md.GoogleTLSConfig
}
switch {
case config.DisableHTTP2 && config.ForceHTTP2:
glog.Fatalf("GAE: DisableHTTP2=%v and ForceHTTP2=%v is conflict!", config.DisableHTTP2, config.ForceHTTP2)
case config.Transport.Proxy.Enabled && config.ForceHTTP2:
glog.Fatalf("GAE: Proxy.Enabled=%v and ForceHTTP2=%v is conflict!", config.Transport.Proxy.Enabled, config.ForceHTTP2)
case config.ForceHTTP2:
tr = &http2.Transport{
DialTLS: md.DialTLS2,
TLSClientConfig: md.GoogleTLSConfig,
DisableCompression: config.Transport.DisableCompression,
}
case !config.DisableHTTP2:
err := http2.ConfigureTransport(t1)
if err != nil {
glog.Warningf("GAE: Error enabling Transport HTTP/2 support: %v", err)
}
tr = t1
default:
tr = t1
}
forceHTTPSMatcherStrings := make([]string, 0)
for key, value := range config.SiteToAlias {
if strings.HasPrefix(value, "google_") {
forceHTTPSMatcherStrings = append(forceHTTPSMatcherStrings, key)
}
}
forceGAEStrings := make([]string, 0)
forceGAESuffixs := make([]string, 0)
forceGAEMatcherStrings := make([]string, 0)
for _, s := range config.ForceGAE {
if strings.Contains(s, "/") {
if strings.HasSuffix(s, "$") {
forceGAESuffixs = append(forceGAESuffixs, strings.TrimRight(s, "$"))
} else {
forceGAEStrings = append(forceGAEStrings, s)
}
} else {
forceGAEMatcherStrings = append(forceGAEMatcherStrings, s)
}
}
if config.EnableDeadProbe && !config.Transport.Proxy.Enabled {
go func() {
probe := func() {
req, _ := http.NewRequest(http.MethodGet, "https://clients3.google.com/generate_204", nil)
ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
defer cancel()
req = req.WithContext(ctx)
resp, err := tr.RoundTrip(req)
if resp != nil && resp.Body != nil {
glog.V(3).Infof("GAE EnableDeadProbe \"%s %s\" %d -", req.Method, req.URL.String(), resp.StatusCode)
resp.Body.Close()
}
if err != nil {
glog.V(2).Infof("GAE EnableDeadProbe \"%s %s\" error: %v", req.Method, req.URL.String(), err)
s := strings.ToLower(err.Error())
if strings.HasPrefix(s, "net/http: request canceled") || strings.Contains(s, "timeout") {
helpers.TryCloseConnections(tr)
}
}
}
for {
time.Sleep(time.Duration(5+rand.Intn(6)) * time.Second)
probe()
}
}()
}
helpers.ShuffleStrings(config.AppIDs)
f := &Filter{
Config: *config,
GAETransport: &Transport{
RoundTripper: tr,
MultiDialer: md,
Servers: NewServers(config.AppIDs,
config.Password,
config.SSLVerify,
time.Duration(config.Transport.ResponseHeaderTimeout-4)*time.Second),
RetryDelay: time.Duration(config.Transport.RetryDelay*1000) * time.Second,
RetryTimes: config.Transport.RetryTimes,
},
DirectTransport: tr,
ForceHTTPSMatcher: helpers.NewHostMatcher(forceHTTPSMatcherStrings),
ForceGAEMatcher: helpers.NewHostMatcher(forceGAEMatcherStrings),
ForceGAEStrings: forceGAEStrings,
ForceGAESuffixs: forceGAESuffixs,
ForceDeflateMatcher: helpers.NewHostMatcher(config.ForceDeflate),
FakeOptionsMatcher: helpers.NewHostMatcherWithStrings(config.FakeOptions),
SiteMatcher: helpers.NewHostMatcher(config.Sites),
DirectSiteMatcher: helpers.NewHostMatcherWithString(config.Site2Alias),
}
if config.Transport.Proxy.Enabled {
f.GAETransport.MultiDialer = nil
}
return f, nil
}
BeforeEach(func() {
defaultTransport = &fakeRoundTripper{}
orgTransport = &fakeRoundTripper{}
comTransport = &fakeRoundTripper{}
transport = statham.NewTransport(defaultTransport, statham.Mapping{
"example.org": orgTransport,
"example.com": comTransport,
})
})
Context("when the host doesn't match any domain in the mapping", func() {
It("uses the default transport", func() {
req, _ := http.NewRequest("GET", "http://example.net", nil)
transport.RoundTrip(req)
Expect(defaultTransport.Calls).To(Equal(1))
Expect(orgTransport.Calls).To(Equal(0))
Expect(comTransport.Calls).To(Equal(0))
})
})
Context("when the host matches a domain in the mapping", func() {
It("uses the corresponding transport", func() {
req, _ := http.NewRequest("GET", "http://example.com", nil)
transport.RoundTrip(req)
Expect(defaultTransport.Calls).To(Equal(0))
Expect(orgTransport.Calls).To(Equal(0))
var err error
fakeEmitter = fake.NewFakeEventEmitter(origin)
fakeRoundTripper = new(FakeRoundTripper)
rt = dropsonde.InstrumentedRoundTripper(fakeRoundTripper, fakeEmitter)
req, err = http.NewRequest("GET", "http://foo.example.com/", nil)
Expect(err).ToNot(HaveOccurred())
req.RemoteAddr = "127.0.0.1"
req.Header.Set("User-Agent", "our-testing-client")
})
Describe("request ID", func() {
It("should generate a new request ID", func() {
rt.RoundTrip(req)
Expect(req.Header.Get("X-CF-RequestID")).ToNot(BeEmpty())
})
Context("if request ID can't be generated", func() {
BeforeEach(func() {
dropsonde.GenerateUuid = func() (u *uuid.UUID, err error) {
return nil, errors.New("test error")
}
})
AfterEach(func() {
dropsonde.GenerateUuid = uuid.NewV4
})
It("defaults to an empty request ID", func() {
rt.RoundTrip(req)
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
var tr http.RoundTripper = f.GAETransport
if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
u := strings.Replace(req.URL.String(), "http://", "https://", 1)
glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
resp := &http.Response{
StatusCode: http.StatusMovedPermanently,
Header: http.Header{
"Location": []string{u},
},
Request: req,
Close: true,
ContentLength: -1,
}
glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if f.DirectSiteMatcher.Match(req.Host) {
switch req.URL.Path {
case "/url":
if rawurl := req.URL.Query().Get("url"); rawurl != "" {
if u, err := url.Parse(rawurl); err == nil {
if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
rawurl = strings.Replace(rawurl, "http://", "https://", 1)
}
}
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
case "/books":
if req.URL.Host == "books.google.cn" {
rawurl := strings.Replace(req.URL.String(), "books.google.cn", "books.google.com", 1)
glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
return ctx, &http.Response{
StatusCode: http.StatusFound,
Header: http.Header{
"Location": []string{rawurl},
},
Request: req,
Close: true,
ContentLength: -1,
}, nil
}
}
if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
tr = f.DirectTransport
if s := req.Header.Get("Connection"); s != "" {
if s1 := strings.ToLower(s); s != s1 {
req.Header.Set("Connection", s1)
}
}
}
}
if tr != f.DirectTransport && req.Method == http.MethodOptions {
if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Request: req,
Close: false,
ContentLength: -1,
}
for _, s := range v.([]string) {
parts := strings.SplitN(s, ":", 2)
if len(parts) == 2 {
resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
}
}
if origin := req.Header.Get("Origin"); origin != "" {
resp.Header.Set("Access-Control-Allow-Origin", origin)
}
if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
resp.Header.Set("Access-Control-Allow-Headers", headers)
}
glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
if tr == f.GAETransport {
switch strings.ToLower(req.Header.Get("Connection")) {
case "upgrade":
resp := &http.Response{
StatusCode: http.StatusForbidden,
Header: http.Header{
"X-WebSocket-Reject-Reason": []string{"Unsupported"},
},
Request: req,
Close: true,
ContentLength: 0,
}
glog.V(2).Infof("%s \"GAE FAKEWEBSOCKET %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, nil
}
}
prefix := "FETCH"
if tr == f.DirectTransport {
prefix = "DIRECT"
}
resp, err := tr.RoundTrip(req)
if err != nil {
glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
if tr == f.DirectTransport {
if ne, ok := err.(interface {
Timeout() bool
}); ok && ne.Timeout() {
// f.MultiDialer.ClearCache()
helpers.CloseConnections(tr)
}
}
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return ctx, nil, err
}
if tr == f.DirectTransport && resp.StatusCode >= http.StatusBadRequest {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
resp.Body.Close()
return ctx, nil, err
}
if addr, err := helpers.ReflectRemoteAddrFromResponse(resp); err == nil {
if ip, _, err := net.SplitHostPort(addr); err == nil {
var duration time.Duration
switch {
case resp.StatusCode == http.StatusBadGateway && bytes.Contains(body, []byte("Please try again in 30 seconds.")):
duration = 1 * time.Hour
case resp.StatusCode == http.StatusNotFound && bytes.Contains(body, []byte("<ins>That’s all we know.</ins>")):
server := resp.Header.Get("Server")
if server != "gws" && !strings.HasPrefix(server, "gvs") {
if md := f.GAETransport.MultiDialer; md != nil && md.TLSConnDuration.Len() > 10 {
duration = 5 * time.Minute
}
}
}
if duration > 0 && f.GAETransport.MultiDialer != nil {
glog.Warningf("GAE: %s StatusCode is %d, not a gws/gvs ip, add to blacklist for %v", ip, resp.StatusCode, duration)
f.GAETransport.MultiDialer.IPBlackList.Set(ip, struct{}{}, time.Now().Add(duration))
if !helpers.CloseConnectionByRemoteAddr(tr, addr) {
glog.Warningf("GAE: CloseConnectionByRemoteAddr(%T, %#v) failed.", tr, addr)
}
}
}
}
resp.Body.Close()
resp.Body = ioutil.NopCloser(bytes.NewReader(body))
}
if resp != nil && resp.Header != nil {
resp.Header.Del("Alt-Svc")
resp.Header.Del("Alternate-Protocol")
}
glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
return ctx, resp, err
}