Beispiel #1
0
func (p *Prometheus) gatherURL(url string, acc telegraf.Accumulator) error {
	collectDate := time.Now()
	var req, err = http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", acceptHeader)
	var token []byte
	var resp *http.Response

	tlsCfg, err := internal.GetTLSConfig(
		p.SSLCert, p.SSLKey, p.SSLCA, p.InsecureSkipVerify)
	if err != nil {
		return err
	}

	var rt http.RoundTripper = &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   5 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout:   5 * time.Second,
		TLSClientConfig:       tlsCfg,
		ResponseHeaderTimeout: time.Duration(3 * time.Second),
		DisableKeepAlives:     true,
	}

	if p.BearerToken != "" {
		token, err = ioutil.ReadFile(p.BearerToken)
		if err != nil {
			return err
		}
		req.Header.Set("Authorization", "Bearer "+string(token))
	}

	resp, err = rt.RoundTrip(req)
	if err != nil {
		return fmt.Errorf("error making HTTP request to %s: %s", url, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%s returned HTTP status %s", url, resp.Status)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("error reading body: %s", err)
	}

	metrics, err := Parse(body, resp.Header)
	if err != nil {
		return fmt.Errorf("error reading metrics for %s: %s",
			url, err)
	}
	// Add (or not) collected metrics
	for _, metric := range metrics {
		tags := metric.Tags()
		tags["url"] = url
		acc.AddFields(metric.Name(), metric.Fields(), tags, collectDate)
	}

	return nil
}
Beispiel #2
0
// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	var transport http.RoundTripper
	if t.Transport != nil {
		transport = t.Transport
	} else {
		transport = http.DefaultTransport
	}

	// To set extra querystring params, we must make a copy of the Request so
	// that we don't modify the Request we were given. This is required by the
	// specification of http.RoundTripper.
	req = cloneRequest(req)

	child := t.Recorder.Child()
	if t.SetName {
		child.Name(req.URL.Host)
	}
	SetSpanIDHeader(req.Header, child.SpanID)

	e := NewClientEvent(req)
	e.ClientSend = time.Now()

	// Make the HTTP request.
	resp, err := transport.RoundTrip(req)

	e.ClientRecv = time.Now()
	if err == nil {
		e.Response = responseInfo(resp)
	} else {
		e.Response.StatusCode = -1
	}
	child.Event(e)

	return resp, err
}
Beispiel #3
0
// Dial dials to the remote url, and sends streaming request. If it succeeds,
// it returns nil error, and the caller should call Handle function to keep
// receiving appendEntry messages.
func (s *streamClient) start(tr http.RoundTripper, u string, cid types.ID) error {
	uu, err := url.Parse(u)
	if err != nil {
		return fmt.Errorf("parse url %s error: %v", u, err)
	}
	uu.Path = path.Join(RaftStreamPrefix, s.id.String())
	req, err := http.NewRequest("GET", uu.String(), nil)
	if err != nil {
		return fmt.Errorf("new request to %s error: %v", u, err)
	}
	req.Header.Set("X-Etcd-Cluster-ID", cid.String())
	req.Header.Set("X-Raft-To", s.to.String())
	req.Header.Set("X-Raft-Term", strconv.FormatUint(s.term, 10))
	resp, err := tr.RoundTrip(req)
	if err != nil {
		return fmt.Errorf("error posting to %q: %v", u, err)
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		return fmt.Errorf("unhandled http status %d", resp.StatusCode)
	}
	s.closer = resp.Body
	go s.handle(resp.Body)
	log.Printf("rafthttp: starting client stream to %s at term %d", s.to, s.term)
	return nil
}
Beispiel #4
0
func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	var u http.RoundTripper
	if t.Transport != nil {
		u = t.Transport
	} else {
		u = http.DefaultTransport
	}

	reqBytes, err := httputil.DumpRequestOut(req, true)
	if err != nil {
		return nil, err
	}
	t.Writer.Write(reqBytes)

	resp, err := u.RoundTrip(req)
	if err != nil {
		return nil, err
	}

	respBytes, err := httputil.DumpResponse(resp, true)
	if err != nil {
		return nil, err
	}
	t.Writer.Write(respBytes)

	return resp, nil
}
Beispiel #5
0
func main() {
	//Set URL to test --- test will NOT follow redirects
	testurl := "https://beacon.walmart.com/vm/ttap.gif?id=10694084&160x600&audience=&creative=1027467&creativetype=rich_media&device=&initiative=209902&placement=SSLTEST&targeting=wmxaudience&vendor=AOD&version=2015802"

	// Set log filename
	logfile := "testlog.txt"

	var DefaultTransport http.RoundTripper = &http.Transport{}

	// Set number of requests
	for i := 0; i < 500000; i++ {

		req, _ := http.NewRequest("GET", testurl, nil)

		resp, _ := DefaultTransport.RoundTrip(req)

		logger, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Fatal(err)
		}

		defer logger.Close()

		// Grab specified headers from HTTP response
		location := resp.Header.Get("Location")
		cookie := resp.Header.Get("Set-Cookie")
		t := time.Now().UTC().String()
		result := fmt.Sprintf("%s\t%s\t%s\r\n", t, location, cookie)
		logger.WriteString(result)

		// Wait time before next request
		time.Sleep(time.Second)
	}
}
Beispiel #6
0
// newStreamClient starts and returns a new started stream client.
// The caller should call stop when finished, to shut it down.
func newStreamReader(id, to, cid types.ID, term uint64, tr http.RoundTripper, u string, r Raft) (*streamReader, error) {
	s := &streamReader{
		id:   id,
		to:   to,
		term: term,
		r:    r,
		done: make(chan struct{}),
	}

	uu, err := url.Parse(u)
	if err != nil {
		return nil, fmt.Errorf("parse url %s error: %v", u, err)
	}
	uu.Path = path.Join(RaftStreamPrefix, s.id.String())
	req, err := http.NewRequest("GET", uu.String(), nil)
	if err != nil {
		return nil, fmt.Errorf("new request to %s error: %v", u, err)
	}
	req.Header.Set("X-Etcd-Cluster-ID", cid.String())
	req.Header.Set("X-Raft-To", s.to.String())
	req.Header.Set("X-Raft-Term", strconv.FormatUint(s.term, 10))
	resp, err := tr.RoundTrip(req)
	if err != nil {
		return nil, fmt.Errorf("error posting to %q: %v", u, err)
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
	}
	s.closer = resp.Body
	go s.handle(resp.Body)
	log.Printf("rafthttp: starting client stream to %s at term %d", s.to, s.term)
	return s, nil
}
Beispiel #7
0
func roundTrip(rt http.RoundTripper, req *http.Request) (body []byte, err error) {
	resp, err := rt.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}
Beispiel #8
0
// ReqThrottledRoundTripper wraps another RoundTripper rt,
// throttling all requests to the specified request rate.
func ReqThrottledRoundTripper(rt http.RoundTripper, rate int64) http.RoundTripper {
	freq := time.Duration(1e9 / rate)
	bucket := tb.NewBucket(rate, freq)

	return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
		got := bucket.Take(1)
		for got != 1 {
			got = bucket.Take(1)
			time.Sleep(freq)
		}
		return rt.RoundTrip(r)
	})
}
Beispiel #9
0
// ByteThrottledRoundTripper wraps another RoundTripper rt,
// throttling all requests to the specified byte rate.
func ByteThrottledRoundTripper(rt http.RoundTripper, rate int64) http.RoundTripper {
	freq := time.Duration(1 * time.Millisecond)
	bucket := tb.NewBucket(rate, freq)

	return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
		got := bucket.Take(r.ContentLength)
		for got < r.ContentLength {
			got += bucket.Take(r.ContentLength - got)
			time.Sleep(freq)
		}
		return rt.RoundTrip(r)
	})
}
Beispiel #10
0
func request(rt http.RoundTripper, requestURL string, requestHeaders http.Header) (*http.Response, error) {
	// Build the request
	req, err := http.NewRequest("GET", requestURL, nil)
	if err != nil {
		return nil, err
	}
	for k, v := range requestHeaders {
		req.Header[k] = v
	}
	req.Header.Set(CSRFTokenHeader, "1")

	// Make the request
	return rt.RoundTrip(req)
}
Beispiel #11
0
// HTTPGet creates a new http request
func HTTPGet(url string, follow, insecure bool, h map[string]string, timeout ...int) (*http.Response, error) {
	// timeout in seconds defaults to 5
	var t int = 5

	if len(timeout) > 0 {
		t = timeout[0]
	}

	// if insecure = true, skip ssl verification
	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout:   10 * time.Second,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: insecure},
		ResponseHeaderTimeout: time.Duration(t) * time.Second,
	}

	client := &http.Client{}
	client.Transport = tr

	// create a new request
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "epazote")

	// set custom headers on request
	if h != nil {
		for k, v := range h {
			req.Header.Set(k, v)
		}
	}

	if follow {
		res, err := client.Do(req)
		if err != nil {
			return nil, err
		}
		return res, nil
	}

	// not follow redirects
	var DefaultTransport http.RoundTripper = tr

	res, err := DefaultTransport.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	return res, nil
}
Beispiel #12
0
func TLSClientConfig(transport http.RoundTripper) (*tls.Config, error) {
	if transport == nil {
		return nil, nil
	}

	switch transport := transport.(type) {
	case *http.Transport:
		return transport.TLSClientConfig, nil
	case RoundTripperWrapper:
		return TLSClientConfig(transport.WrappedRoundTripper())
	default:
		return nil, fmt.Errorf("unknown transport type: %v", transport)
	}
}
Beispiel #13
0
func Dialer(transport http.RoundTripper) (DialFunc, error) {
	if transport == nil {
		return nil, nil
	}

	switch transport := transport.(type) {
	case *http.Transport:
		return transport.Dial, nil
	case RoundTripperWrapper:
		return Dialer(transport.WrappedRoundTripper())
	default:
		return nil, fmt.Errorf("unknown transport type: %v", transport)
	}
}
Beispiel #14
0
func (S) TestTransport(c *gocheck.C) {
	var t http.RoundTripper = &Transport{
		Message: "Ok",
		Status:  http.StatusOK,
		Headers: map[string][]string{"Authorization": {"something"}},
	}
	req, _ := http.NewRequest("GET", "/", nil)
	r, err := t.RoundTrip(req)
	c.Assert(err, gocheck.IsNil)
	c.Assert(r.StatusCode, gocheck.Equals, http.StatusOK)
	defer r.Body.Close()
	b, _ := ioutil.ReadAll(r.Body)
	c.Assert(string(b), gocheck.Equals, "Ok")
	c.Assert(r.Header.Get("Authorization"), gocheck.Equals, "something")
}
Beispiel #15
0
// NewRoundTripper creates an http.RoundTripper to instrument external requests.
// This RoundTripper must be used in same the goroutine as the other uses of the
// Transaction's SegmentTracer methods.  http.DefaultTransport is used if an
// http.RoundTripper is not provided.
//
//   client := &http.Client{}
//   client.Transport = newrelic.NewRoundTripper(txn, nil)
//   resp, err := client.Get("http://example.com/")
//
func NewRoundTripper(txn Transaction, original http.RoundTripper) http.RoundTripper {
	return roundTripperFunc(func(request *http.Request) (*http.Response, error) {
		segment := StartExternalSegment(txn, request)

		if nil == original {
			original = http.DefaultTransport
		}
		response, err := original.RoundTrip(request)

		segment.Response = response
		segment.End()

		return response, err
	})
}
Beispiel #16
0
// NewHTTPPoolTransport returns a function that must be used in groupcache.HTTPPool.Transport.
//
// rt is optional, http.DefaultTransport is used by default.
func NewHTTPPoolTransport(rt http.RoundTripper) func(groupcache.Context) http.RoundTripper {
	if rt == nil {
		rt = http.DefaultTransport
	}
	return func(ctx groupcache.Context) http.RoundTripper {
		return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
			if ctx, ok := ctx.(*Context); ok && ctx != nil {
				err := setContext(req, ctx)
				if err != nil {
					return nil, err
				}
			}
			return rt.RoundTrip(req)
		})
	}
}
Beispiel #17
0
// RoundTrip implements the http.RoundTripper interface.
//
// For higher-level HTTP client support (such as handling of cookies
// and redirects), see Get, Post, and the Client type.
func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	if req.URL == nil {
		closeBody(req)
		return nil, errors.New("http: nil Request.URL")
	}
	if req.Header == nil {
		closeBody(req)
		return nil, errors.New("http: nil Request.Header")
	}
	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
		t.altMu.RLock()
		var rt http.RoundTripper
		if t.altProto != nil {
			rt = t.altProto[req.URL.Scheme]
		}
		t.altMu.RUnlock()
		if rt == nil {
			closeBody(req)
			return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
		}
		return rt.RoundTrip(req)
	}
	if req.URL.Host == "" {
		closeBody(req)
		return nil, errors.New("http: no Host in request URL")
	}
	treq := &transportRequest{Request: req}
	cm, err := t.connectMethodForRequest(treq)
	if err != nil {
		closeBody(req)
		return nil, err
	}

	// Get the cached or newly-created connection to either the
	// host (for http or https), the http proxy, or the http proxy
	// pre-CONNECTed to https server.  In any case, we'll be ready
	// to send it requests.
	pconn, err := t.getConn(req, cm)
	if err != nil {
		t.setReqCanceler(req, nil)
		closeBody(req)
		return nil, err
	}

	return pconn.roundTrip(treq)
}
Beispiel #18
0
// RoundTrip implements the RoundTripper interface.
func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	var transport http.RoundTripper
	if t.Transport != nil {
		transport = t.Transport
	} else {
		transport = http.DefaultTransport
	}

	// To set extra querystring params, we must make a copy of the Request so
	// that we don't modify the Request we were given. This is required by the
	// specification of http.RoundTripper.
	req = cloneRequest(req)
	req.SetBasicAuth(t.Username, t.Password)

	// Make the HTTP request.
	return transport.RoundTrip(req)
}
Beispiel #19
0
func downloadFile(d *drive.Service, t http.RoundTripper, f *drive.File) {
	downloadUrl := f.DownloadUrl

	if downloadUrl == "" {
		log.Fatal("An error occurred: File is not downloadable")
	}

	req, _ := http.NewRequest("GET", downloadUrl, nil)

	resp, _ := t.RoundTrip(req)
	defer resp.Body.Close()

	file, _ := os.Create(eventFilePath)
	defer file.Close()

	if _, err := io.Copy(file, resp.Body); err != nil {
		log.Fatal(err)
	}
}
Beispiel #20
0
func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	var transport http.RoundTripper
	if t.Transport != nil {
		transport = t.Transport
	} else {
		transport = http.DefaultTransport
	}
	req2 := SetHeaders(req, t.Span)

	event := NewEvent(t.Span, ClientRequestSent)
	t.Collector.Record(event)

	resp, err := transport.RoundTrip(req2)

	event = NewEvent(t.Span, ClientRequestReceived)
	t.Collector.Record(event)

	return resp, err
}
Beispiel #21
0
// Accepts a Httpaction and a one-way channel to write the results to.
func DoHttpRequest(httpAction HttpAction, resultsChannel chan HttpReqResult, sessionMap map[string]string) {
	req := buildHttpRequest(httpAction, sessionMap)

	start := time.Now()
	var DefaultTransport http.RoundTripper = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	resp, err := DefaultTransport.RoundTrip(req)

	if err != nil {
		log.Printf("HTTP request failed: %s", err)
	} else {
		elapsed := time.Since(start)
		responseBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			//log.Fatal(err)
			log.Printf("Reading HTTP response failed: %s\n", err)
			httpReqResult := buildHttpResult(0, resp.StatusCode, elapsed.Nanoseconds(), httpAction.Title)

			resultsChannel <- httpReqResult
		} else {
			defer resp.Body.Close()

			if httpAction.StoreCookie != "" {
				for _, cookie := range resp.Cookies() {

					if cookie.Name == httpAction.StoreCookie {
						sessionMap["____"+cookie.Name] = cookie.Value
					}
				}
			}

			// if action specifies response action, parse using regexp/jsonpath
			processResult(httpAction, sessionMap, responseBody)

			httpReqResult := buildHttpResult(len(responseBody), resp.StatusCode, elapsed.Nanoseconds(), httpAction.Title)

			resultsChannel <- httpReqResult
		}
	}
}
Beispiel #22
0
// RoundTrip implements http.RoundTripper.
func (t *TicketAuthedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	var transport http.RoundTripper
	if t.Transport != nil {
		transport = t.Transport
	} else {
		transport = http.DefaultTransport
	}

	if len(t.SignedTicketStrings) > 0 {
		// To set extra headers, we must make a copy of the Request so
		// that we don't modify the Request we were given. This is
		// required by the specification of http.RoundTripper.
		req = cloneRequest(req)
		for _, tstr := range t.SignedTicketStrings {
			req.Header.Add("authorization", TicketAuthScheme+tstr)
		}
	}

	// Make the HTTP request.
	return transport.RoundTrip(req)
}
Beispiel #23
0
func (S) TestConditionalTransport(c *gocheck.C) {
	var t http.RoundTripper = &ConditionalTransport{
		Transport: Transport{
			Message: "Ok",
			Status:  http.StatusOK,
		},
		CondFunc: func(req *http.Request) bool {
			return req.URL.Path == "/something"
		},
	}
	req, _ := http.NewRequest("GET", "/something", nil)
	r, err := t.RoundTrip(req)
	c.Assert(err, gocheck.IsNil)
	c.Assert(r.StatusCode, gocheck.Equals, http.StatusOK)
	defer r.Body.Close()
	b, _ := ioutil.ReadAll(r.Body)
	c.Assert(string(b), gocheck.Equals, "Ok")
	req, _ = http.NewRequest("GET", "/", nil)
	r, err = t.RoundTrip(req)
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "condition failed")
	c.Assert(r.StatusCode, gocheck.Equals, http.StatusInternalServerError)
}
Beispiel #24
0
func benchmarkEchoProtoHTTP(b *testing.B, size int, accept func(net.Listener, *tls.Config) error, roundTripper http.RoundTripper) {
	var url string
	benchmarkEcho(b, size, accept,
		func(addr net.Addr) {
			url = fmt.Sprintf("https://%s", addr)
		},
		nil,
		func(echoMsg string) string {
			args := EchoRequest{Msg: echoMsg}
			reqBody, err := proto.Marshal(&args)
			if err != nil {
				b.Fatal(err)
			}
			req, err := http.NewRequest("POST", url, bytes.NewReader(reqBody))
			if err != nil {
				b.Fatal(err)
			}
			req.Header.Set("Content-Type", xProtobuf)
			resp, err := roundTripper.RoundTrip(req)
			if err != nil {
				b.Fatal(err)
			}
			respBody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				b.Fatal(err)
			}
			if err := resp.Body.Close(); err != nil {
				b.Fatal(err)
			}
			reply := EchoResponse{}
			if err := proto.Unmarshal(respBody, &reply); err != nil {
				b.Fatal(err)
			}
			return reply.Msg
		},
	)
}
Beispiel #25
0
type fakeRoundTripper struct {
	Calls int
}

func (rt *fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	rt.Calls++

	return nil, nil
}

var _ = Describe("A per-host transport mapping", func() {
	var (
		defaultTransport *fakeRoundTripper
		orgTransport     *fakeRoundTripper
		comTransport     *fakeRoundTripper

		transport http.RoundTripper
	)

	BeforeEach(func() {
		defaultTransport = &fakeRoundTripper{}
		orgTransport = &fakeRoundTripper{}
		comTransport = &fakeRoundTripper{}

		transport = statham.NewTransport(defaultTransport, statham.Mapping{
			"example.org": orgTransport,
			"example.com": comTransport,
		})
	})
Beispiel #26
0
func (h ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	activeConnections.Add(1)
	defer activeConnections.Done()

	conf := GetConfig()

	if !conf.ACLsLoaded {
		http.Error(w, "Redwood proxy configuration needs to be updated for this version of Redwood.\n(Use ACLs)", 500)
		return
	}

	if len(r.URL.String()) > 10000 {
		http.Error(w, "URL too long", http.StatusRequestURITooLong)
		return
	}

	client := r.RemoteAddr
	host, _, err := net.SplitHostPort(client)
	if err == nil {
		client = host
	}

	if conf.AuthCacheTime > 0 {
		auth := r.Header.Get("Proxy-Authorization")
		if auth == "" {
			authCacheLock.RLock()
			ar, ok := authCache[client]
			authCacheLock.RUnlock()
			if ok && time.Now().Sub(ar.Time) < time.Duration(conf.AuthCacheTime)*time.Second {
				r.Header.Set("Proxy-Authorization", ar.ProxyAuthorization)
			}
		} else {
			authCacheLock.Lock()
			authCache[client] = authRecord{
				ProxyAuthorization: auth,
				Time:               time.Now(),
			}
			authCacheLock.Unlock()
		}
	}

	if r.Header.Get("Proxy-Authorization") != "" {
		user, pass := ProxyCredentials(r)
		if !conf.ValidCredentials(user, pass) {
			log.Printf("Incorrect username or password from %v: %q:%q", r.RemoteAddr, user, pass)
			r.Header.Del("Proxy-Authorization")
		}
	}

	// Reconstruct the URL if it is incomplete (i.e. on a transparent proxy).
	if r.URL.Host == "" {
		r.URL.Host = r.Host
	}
	if r.URL.Scheme == "" {
		if h.TLS {
			r.URL.Scheme = "https"
		} else {
			r.URL.Scheme = "http"
		}
	}

	var userAgent string
	if conf.LogUserAgent {
		userAgent = r.Header.Get("User-Agent")
	}

	if realHost, ok := conf.VirtualHosts[r.Host]; ok {
		r.Host = realHost
		r.URL.Host = realHost
	}

	user := client
	var authUser string
	if h.user != "" {
		authUser = h.user
	} else if u, _ := ProxyCredentials(r); u != "" {
		authUser = u
	}
	if authUser != "" {
		user = authUser
	}

	tally := conf.URLRules.MatchingRules(r.URL)
	scores := conf.categoryScores(tally)
	categories := conf.significantCategories(scores)

	reqACLs := conf.ACLs.requestACLs(r, authUser)

	possibleActions := []string{
		"allow",
		"block",
		"block-invisible",
	}
	if r.Header.Get("Proxy-Authorization") == "" && !h.TLS {
		possibleActions = append(possibleActions, "require-auth")
	}
	if r.Method == "CONNECT" && conf.TLSReady {
		possibleActions = append(possibleActions, "ssl-bump")
	}

	thisRule, ignored := conf.ChooseACLCategoryAction(reqACLs, categories, possibleActions...)
	if r.Method == "CONNECT" && conf.TLSReady && thisRule.Action == "" {
		// If the result is unclear, go ahead and start to bump the connection.
		// The ACLs will be checked one more time anyway.
		thisRule.Action = "ssl-bump"
	}

	switch thisRule.Action {
	case "require-auth":
		conf.send407(w)
		log.Printf("Missing required proxy authentication from %v to %v", r.RemoteAddr, r.URL)
		return
	case "block":
		conf.showBlockPage(w, r, user, tally, scores, thisRule)
		logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
		return
	case "block-invisible":
		showInvisibleBlock(w)
		logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
		return
	case "ssl-bump":
		conn, err := newHijackedConn(w)
		if err != nil {
			fmt.Fprintln(conn, "HTTP/1.1 500 Internal Server Error")
			fmt.Fprintln(conn)
			fmt.Fprintln(conn, err)
			conn.Close()
			return
		}
		fmt.Fprint(conn, "HTTP/1.1 200 Connection Established\r\n\r\n")
		SSLBump(conn, r.URL.Host, user, authUser)
		return
	}

	if r.Host == localServer {
		conf.ServeMux.ServeHTTP(w, r)
		return
	}

	if r.Method == "CONNECT" {
		conn, err := newHijackedConn(w)
		if err != nil {
			fmt.Fprintln(conn, "HTTP/1.1 500 Internal Server Error")
			fmt.Fprintln(conn)
			fmt.Fprintln(conn, err)
			conn.Close()
			return
		}
		fmt.Fprint(conn, "HTTP/1.1 200 Connection Established\r\n\r\n")
		logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
		connectDirect(conn, r.URL.Host, nil)
		return
	}

	if r.Header.Get("Upgrade") == "websocket" {
		h.makeWebsocketConnection(w, r)
		return
	}

	r.Header.Add("Via", r.Proto+" Redwood")
	r.Header.Add("X-Forwarded-For", client)

	gzipOK := !conf.DisableGZIP && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && !lanAddress(client)
	r.Header.Del("Accept-Encoding")

	urlChanged := conf.changeQuery(r.URL)

	if !urlChanged {
		// Rebuild the URL in a way that will preserve which characters are escaped
		// and which aren't, for compatibility with broken servers.
		rawURL := r.RequestURI
		if strings.HasPrefix(rawURL, r.URL.Scheme) {
			rawURL = rawURL[len(r.URL.Scheme):]
			rawURL = strings.TrimPrefix(rawURL, "://")
			slash := strings.Index(rawURL, "/")
			if slash == -1 {
				rawURL = "/"
			} else {
				rawURL = rawURL[slash:]
			}
		}
		q := strings.Index(rawURL, "?")
		if q != -1 {
			rawURL = rawURL[:q]
		}
		if strings.HasPrefix(rawURL, "//") {
			// The path should start with a single slash not two.
			rawURL = rawURL[1:]
		}
		r.URL.Opaque = rawURL
	}

	proxied := false
	var rt http.RoundTripper
	if h.rt == nil {
		if r.URL.Opaque != "" && transport.Proxy != nil {
			if p, _ := transport.Proxy(r); p != nil {
				// If the request is going through a proxy, the host needs to be
				// included in the opaque element.
				r.URL.Opaque = "//" + r.URL.Host + r.URL.Opaque
				proxied = true
			}
		}
		rt = &transport
	} else {
		rt = h.rt
	}

	if !proxied {
		r.Header.Del("Proxy-Authorization")
	}
	resp, err := rt.RoundTrip(r)
	r.URL.Opaque = ""

	if err != nil {
		http.Error(w, err.Error(), http.StatusServiceUnavailable)
		log.Printf("error fetching %s: %s", r.URL, err)
		logAccess(r, nil, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
		return
	}
	defer resp.Body.Close()

	// Prevent switching to QUIC.
	resp.Header.Del("Alternate-Protocol")

	originalContentType := resp.Header.Get("Content-Type")
	fixContentType(resp)

	respACLs := conf.ACLs.responseACLs(resp)
	acls := unionACLSets(reqACLs, respACLs)
	thisRule, ignored = conf.ChooseACLCategoryAction(acls, categories, "allow", "block", "block-invisible", "hash-image", "phrase-scan")
	if thisRule.Action == "" {
		thisRule.Action = "allow"
	}

	switch thisRule.Action {
	case "allow":
		resp.Header.Set("Content-Type", originalContentType)
		copyResponseHeader(w, resp)
		n, err := io.Copy(w, resp.Body)
		if err != nil {
			log.Printf("error while copying response (URL: %s): %s", r.URL, err)
		}
		logAccess(r, resp, int(n), false, user, tally, scores, thisRule, "", ignored, userAgent)
		return
	case "block":
		conf.showBlockPage(w, r, user, tally, scores, thisRule)
		logAccess(r, resp, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
		return
	case "block-invisible":
		showInvisibleBlock(w)
		logAccess(r, resp, 0, false, user, tally, scores, thisRule, "", ignored, userAgent)
		return
	}

	lr := &io.LimitedReader{
		R: resp.Body,
		N: 1e6,
	}
	content, err := ioutil.ReadAll(lr)
	if err != nil {
		log.Printf("error while reading response body (URL: %s): %s", r.URL, err)
	}
	if lr.N == 0 {
		log.Println("response body too long to filter:", r.URL)
		resp.Header.Set("Content-Type", originalContentType)
		var dest io.Writer = w
		if gzipOK {
			resp.Header.Set("Content-Encoding", "gzip")
			resp.Header.Del("Content-Length")
			gzw := gzip.NewWriter(w)
			defer gzw.Close()
			dest = gzw
		}
		copyResponseHeader(w, resp)
		dest.Write(content)
		n, err := io.Copy(dest, resp.Body)
		if err != nil {
			log.Printf("error while copying response (URL: %s): %s", r.URL, err)
		}
		logAccess(r, resp, int(n)+len(content), false, user, tally, scores, ACLActionRule{Action: "allow", Needed: []string{"too-long-to-filter"}}, "", ignored, userAgent)
		return
	}

	modified := false
	pageTitle := ""

	switch thisRule.Action {
	case "phrase-scan":
		contentType := resp.Header.Get("Content-Type")
		_, cs, _ := charset.DetermineEncoding(content, contentType)
		if strings.Contains(contentType, "html") {
			var doc *html.Node
			if conf.LogTitle {
				doc, err = parseHTML(content, cs)
				if err != nil {
					log.Printf("Error parsing HTML from %s: %s", r.URL, err)
				} else {
					t := titleSelector.MatchFirst(doc)
					if t != nil {
						if titleText := t.FirstChild; titleText != nil && titleText.Type == html.TextNode {
							pageTitle = titleText.Data
						}
					}
				}
			}

			modified = conf.pruneContent(r.URL, &content, cs, acls, doc)
			if modified {
				resp.Header.Set("Content-Type", "text/html; charset=utf-8")
				cs = "utf-8"
				resp.Header.Del("Content-Length")
			}
		}

		conf.scanContent(content, contentType, cs, tally)

	case "hash-image":
		img, _, err := image.Decode(bytes.NewReader(content))
		if err != nil {
			log.Printf("Error decoding image from %v: %v", r.URL, err)
			break
		}
		hash := dhash.New(img)

		for _, h := range conf.ImageHashes {
			if dhash.Distance(hash, h) <= conf.DhashThreshold {
				tally[rule{imageHash, h.String()}]++
			}
		}
	}

	scores = conf.categoryScores(tally)
	categories = conf.significantCategories(scores)
	thisRule, ignored = conf.ChooseACLCategoryAction(acls, categories, "allow", "block", "block-invisible")
	if thisRule.Action == "" {
		thisRule.Action = "allow"
	}

	switch thisRule.Action {
	case "block":
		conf.showBlockPage(w, r, user, tally, scores, thisRule)
		logAccess(r, resp, len(content), modified, user, tally, scores, thisRule, pageTitle, ignored, userAgent)
		return
	case "block-invisible":
		showInvisibleBlock(w)
		logAccess(r, resp, len(content), modified, user, tally, scores, thisRule, pageTitle, ignored, userAgent)
		return
	}

	if !modified {
		resp.Header.Set("Content-Type", originalContentType)
	}

	if gzipOK && len(content) > 1000 {
		resp.Header.Set("Content-Encoding", "gzip")
		resp.Header.Del("Content-Length")
		copyResponseHeader(w, resp)
		gzw := gzip.NewWriter(w)
		gzw.Write(content)
		gzw.Close()
	} else {
		copyResponseHeader(w, resp)
		w.Write(content)
	}

	logAccess(r, resp, len(content), modified, user, tally, scores, thisRule, pageTitle, ignored, userAgent)
}
	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type nullVarz struct{}

var _ = Describe("ProxyRoundTripper", func() {
	Context("RoundTrip", func() {
		var (
			proxyRoundTripper http.RoundTripper
			endpointIterator  *routefakes.FakeEndpointIterator
			transport         *roundtripperfakes.FakeRoundTripper
			logger            lager.Logger
			req               *http.Request
			resp              *proxyfakes.FakeProxyResponseWriter
			dialError         = &net.OpError{
				Err: errors.New("error"),
				Op:  "dial",
			}
		)

		BeforeEach(func() {
			endpointIterator = &routefakes.FakeEndpointIterator{}
			req = test_util.NewRequest("GET", "myapp.com", "/", nil)
			req.URL.Scheme = "http"
			resp = &proxyfakes.FakeProxyResponseWriter{}

			logger = lagertest.NewTestLogger("test")
			transport = &roundtripperfakes.FakeRoundTripper{}
		})
Beispiel #28
0
func NewFilter(config *Config) (filters.Filter, error) {
	dnsServers := make([]net.IP, 0)
	for _, s := range config.DNSServers {
		if ip := net.ParseIP(s); ip != nil {
			dnsServers = append(dnsServers, ip)
		}
	}

	GoogleG2PKP, err := base64.StdEncoding.DecodeString(config.GoogleG2PKP)
	if err != nil {
		return nil, err
	}

	googleTLSConfig := &tls.Config{
		MinVersion:         tls.VersionTLS12,
		InsecureSkipVerify: true,
		ServerName:         "www.microsoft.com",
		ClientSessionCache: tls.NewLRUClientSessionCache(config.TLSConfig.ClientSessionCacheSize),
		CipherSuites: []uint16{
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}
	switch config.TLSConfig.Version {
	case "TLSv12", "TLSv1.2":
		googleTLSConfig.MinVersion = tls.VersionTLS12
	case "TLSv11", "TLSv1.1":
		googleTLSConfig.MinVersion = tls.VersionTLS11
	default:
		googleTLSConfig.MinVersion = tls.VersionTLS10
	}
	pickupCiphers := func(names []string) []uint16 {
		ciphers := make([]uint16, 0)
		for _, name := range names {
			cipher := helpers.Cipher(name)
			if cipher == 0 {
				glog.Fatalf("GAE: cipher %#v is not supported.", name)
			}
			ciphers = append(ciphers, cipher)
		}
		helpers.ShuffleUint16s(ciphers)
		ciphers = ciphers[:1+rand.Intn(len(ciphers))]
		ciphers1 := []uint16{}
		for _, name := range []string{
			"TLS_RSA_WITH_AES_256_CBC_SHA256",
			"TLS_RSA_WITH_AES_256_GCM_SHA384",
			"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
			"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
		} {
			if !helpers.ContainsString(names, name) {
				if c := helpers.Cipher(name); c != 0 {
					ciphers1 = append(ciphers1, c)
				}
			}
		}
		helpers.ShuffleUint16s(ciphers1)
		ciphers1 = ciphers1[:rand.Intn(len(ciphers1))]
		ciphers = append(ciphers, ciphers1...)
		helpers.ShuffleUint16s(ciphers)
		return ciphers
	}
	googleTLSConfig.CipherSuites = pickupCiphers(config.TLSConfig.Ciphers)
	if len(config.TLSConfig.ServerName) > 0 {
		googleTLSConfig.ServerName = config.TLSConfig.ServerName[rand.Intn(len(config.TLSConfig.ServerName))]
	}
	if !config.DisableHTTP2 {
		googleTLSConfig.NextProtos = []string{"h2", "h2-14", "http/1.1"}
	}

	if config.Site2Alias == nil {
		config.Site2Alias = make(map[string]string)
	}
	for key, value := range config.SiteToAlias {
		config.Site2Alias[key] = value
	}
	config.SiteToAlias = config.Site2Alias

	hostmap := map[string][]string{}
	for key, value := range config.HostMap {
		hostmap[key] = helpers.UniqueStrings(value)
	}

	d := &net.Dialer{
		KeepAlive: time.Duration(config.Transport.Dialer.KeepAlive) * time.Second,
		Timeout:   time.Duration(config.Transport.Dialer.Timeout) * time.Second,
		DualStack: config.Transport.Dialer.DualStack,
	}

	md := &dialer.MultiDialer{
		Dialer:            d,
		DisableIPv6:       config.DisableIPv6,
		ForceIPv6:         config.ForceIPv6,
		SSLVerify:         config.SSLVerify,
		EnableRemoteDNS:   config.EnableRemoteDNS,
		LogToStderr:       flag.Lookup("logtostderr") != nil,
		TLSConfig:         nil,
		SiteToAlias:       helpers.NewHostMatcherWithString(config.SiteToAlias),
		IPBlackList:       lrucache.NewLRUCache(1024),
		HostMap:           hostmap,
		GoogleTLSConfig:   googleTLSConfig,
		GoogleG2PKP:       GoogleG2PKP,
		DNSServers:        dnsServers,
		DNSCache:          lrucache.NewLRUCache(config.Transport.Dialer.DNSCacheSize),
		DNSCacheExpiry:    time.Duration(config.Transport.Dialer.DNSCacheExpiry) * time.Second,
		TLSConnDuration:   lrucache.NewLRUCache(8192),
		TLSConnError:      lrucache.NewLRUCache(8192),
		TLSConnReadBuffer: config.Transport.Dialer.SocketReadBuffer,
		ConnExpiry:        5 * time.Minute,
		Level:             config.Transport.Dialer.Level,
	}

	for _, ip := range config.IPBlackList {
		md.IPBlackList.Set(ip, struct{}{}, time.Time{})
	}

	var tr http.RoundTripper

	t1 := &http.Transport{
		Dial:                  md.Dial,
		DialTLS:               md.DialTLS,
		DisableKeepAlives:     config.Transport.DisableKeepAlives,
		DisableCompression:    config.Transport.DisableCompression,
		ResponseHeaderTimeout: time.Duration(config.Transport.ResponseHeaderTimeout) * time.Second,
		IdleConnTimeout:       time.Duration(config.Transport.IdleConnTimeout) * time.Second,
		MaxIdleConnsPerHost:   config.Transport.MaxIdleConnsPerHost,
	}

	if config.Transport.Proxy.Enabled {
		fixedURL, err := url.Parse(config.Transport.Proxy.URL)
		if err != nil {
			glog.Fatalf("url.Parse(%#v) error: %s", config.Transport.Proxy.URL, err)
		}

		dialer, err := proxy.FromURL(fixedURL, d, &dialer.MultiResolver{md})
		if err != nil {
			glog.Fatalf("proxy.FromURL(%#v) error: %s", fixedURL.String(), err)
		}

		t1.Dial = dialer.Dial
		t1.DialTLS = nil
		t1.Proxy = nil
		t1.TLSClientConfig = md.GoogleTLSConfig
	}

	switch {
	case config.DisableHTTP2 && config.ForceHTTP2:
		glog.Fatalf("GAE: DisableHTTP2=%v and ForceHTTP2=%v is conflict!", config.DisableHTTP2, config.ForceHTTP2)
	case config.Transport.Proxy.Enabled && config.ForceHTTP2:
		glog.Fatalf("GAE: Proxy.Enabled=%v and ForceHTTP2=%v is conflict!", config.Transport.Proxy.Enabled, config.ForceHTTP2)
	case config.ForceHTTP2:
		tr = &http2.Transport{
			DialTLS:            md.DialTLS2,
			TLSClientConfig:    md.GoogleTLSConfig,
			DisableCompression: config.Transport.DisableCompression,
		}
	case !config.DisableHTTP2:
		err := http2.ConfigureTransport(t1)
		if err != nil {
			glog.Warningf("GAE: Error enabling Transport HTTP/2 support: %v", err)
		}
		tr = t1
	default:
		tr = t1
	}

	forceHTTPSMatcherStrings := make([]string, 0)
	for key, value := range config.SiteToAlias {
		if strings.HasPrefix(value, "google_") {
			forceHTTPSMatcherStrings = append(forceHTTPSMatcherStrings, key)
		}
	}

	forceGAEStrings := make([]string, 0)
	forceGAESuffixs := make([]string, 0)
	forceGAEMatcherStrings := make([]string, 0)
	for _, s := range config.ForceGAE {
		if strings.Contains(s, "/") {
			if strings.HasSuffix(s, "$") {
				forceGAESuffixs = append(forceGAESuffixs, strings.TrimRight(s, "$"))
			} else {
				forceGAEStrings = append(forceGAEStrings, s)
			}
		} else {
			forceGAEMatcherStrings = append(forceGAEMatcherStrings, s)
		}
	}

	if config.EnableDeadProbe && !config.Transport.Proxy.Enabled {
		go func() {
			probe := func() {
				req, _ := http.NewRequest(http.MethodGet, "https://clients3.google.com/generate_204", nil)
				ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
				defer cancel()
				req = req.WithContext(ctx)
				resp, err := tr.RoundTrip(req)
				if resp != nil && resp.Body != nil {
					glog.V(3).Infof("GAE EnableDeadProbe \"%s %s\" %d -", req.Method, req.URL.String(), resp.StatusCode)
					resp.Body.Close()
				}
				if err != nil {
					glog.V(2).Infof("GAE EnableDeadProbe \"%s %s\" error: %v", req.Method, req.URL.String(), err)
					s := strings.ToLower(err.Error())
					if strings.HasPrefix(s, "net/http: request canceled") || strings.Contains(s, "timeout") {
						helpers.TryCloseConnections(tr)
					}
				}
			}

			for {
				time.Sleep(time.Duration(5+rand.Intn(6)) * time.Second)
				probe()
			}
		}()
	}

	helpers.ShuffleStrings(config.AppIDs)

	f := &Filter{
		Config: *config,
		GAETransport: &Transport{
			RoundTripper: tr,
			MultiDialer:  md,
			Servers: NewServers(config.AppIDs,
				config.Password,
				config.SSLVerify,
				time.Duration(config.Transport.ResponseHeaderTimeout-4)*time.Second),
			RetryDelay: time.Duration(config.Transport.RetryDelay*1000) * time.Second,
			RetryTimes: config.Transport.RetryTimes,
		},
		DirectTransport:     tr,
		ForceHTTPSMatcher:   helpers.NewHostMatcher(forceHTTPSMatcherStrings),
		ForceGAEMatcher:     helpers.NewHostMatcher(forceGAEMatcherStrings),
		ForceGAEStrings:     forceGAEStrings,
		ForceGAESuffixs:     forceGAESuffixs,
		ForceDeflateMatcher: helpers.NewHostMatcher(config.ForceDeflate),
		FakeOptionsMatcher:  helpers.NewHostMatcherWithStrings(config.FakeOptions),
		SiteMatcher:         helpers.NewHostMatcher(config.Sites),
		DirectSiteMatcher:   helpers.NewHostMatcherWithString(config.Site2Alias),
	}

	if config.Transport.Proxy.Enabled {
		f.GAETransport.MultiDialer = nil
	}

	return f, nil
}
Beispiel #29
0
// GetRepository yields a repository matching the given name, if any exists.
// Repository may be of various forms, in which case omitted elements take
// assumed defaults.
//
//   helloworld             -> index.docker.io/library/helloworld
//   foo/helloworld         -> index.docker.io/foo/helloworld
//   quay.io/foo/helloworld -> quay.io/foo/helloworld
//
func (c *Client) GetRepository(repository string) ([]flux.ImageDescription, error) {
	var host, org, image string
	parts := strings.Split(repository, "/")
	switch len(parts) {
	case 1:
		host = dockerHubHost
		org = dockerHubLibrary
		image = parts[0]
	case 2:
		host = dockerHubHost
		org = parts[0]
		image = parts[1]
	case 3:
		host = parts[0]
		org = parts[1]
		image = parts[2]
	default:
		return nil, fmt.Errorf(`expected image name as either "<host>/<org>/<image>", "<org>/<image>", or "<image>"`)
	}

	hostlessImageName := fmt.Sprintf("%s/%s", org, image)
	httphost := "https://" + host

	// quay.io wants us to use cookies for authorisation, so we have
	// to construct one (the default client has none). This means a
	// bit more constructing things to be able to make a registry
	// client literal, rather than calling .New()
	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
	if err != nil {
		return nil, err
	}
	auth := c.Credentials.credsFor(host)

	// A context we'll use to cancel requests on error
	ctx, cancel := context.WithCancel(context.Background())

	// Use the wrapper to fix headers for quay.io, and remember bearer tokens
	var transport http.RoundTripper = &wwwAuthenticateFixer{transport: http.DefaultTransport}
	// Now the auth-handling wrappers that come with the library
	transport = dockerregistry.WrapTransport(transport, httphost, auth.username, auth.password)

	client := &dockerregistry.Registry{
		URL: httphost,
		Client: &http.Client{
			Transport: roundtripperFunc(func(r *http.Request) (*http.Response, error) {
				return transport.RoundTrip(r.WithContext(ctx))
			}),
			Jar: jar,
		},
		Logf: dockerregistry.Quiet,
	}

	tags, err := client.Tags(hostlessImageName)
	if err != nil {
		cancel()
		return nil, err
	}

	// the hostlessImageName is canonicalised, in the sense that it
	// includes "library" as the org, if unqualified -- e.g.,
	// `library/nats`. We need that to fetch the tags etc. However, we
	// want the results to use the *actual* name of the images to be
	// as supplied, e.g., `nats`.
	return c.tagsToRepository(cancel, client, hostlessImageName, repository, tags)
}
Beispiel #30
0
func (f *Filter) RoundTrip(ctx context.Context, req *http.Request) (context.Context, *http.Response, error) {
	if !f.SiteMatcher.Match(req.Host) {
		return ctx, nil, nil
	}

	var tr http.RoundTripper = f.GAETransport

	if req.URL.Scheme == "http" && f.ForceHTTPSMatcher.Match(req.Host) {
		if !strings.HasPrefix(req.Header.Get("Referer"), "https://") {
			u := strings.Replace(req.URL.String(), "http://", "https://", 1)
			glog.V(2).Infof("GAE FORCEHTTPS get raw url=%v, redirect to %v", req.URL.String(), u)
			resp := &http.Response{
				StatusCode: http.StatusMovedPermanently,
				Header: http.Header{
					"Location": []string{u},
				},
				Request:       req,
				Close:         true,
				ContentLength: -1,
			}
			glog.V(2).Infof("%s \"GAE FORCEHTTPS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	if f.DirectSiteMatcher.Match(req.Host) {
		if req.URL.Path == "/url" {
			if rawurl := req.URL.Query().Get("url"); rawurl != "" {
				if u, err := url.Parse(rawurl); err == nil {
					if u.Scheme == "http" && f.ForceHTTPSMatcher.Match(u.Host) {
						rawurl = strings.Replace(rawurl, "http://", "https://", 1)
					}
				}
				glog.V(2).Infof("%s \"GAE REDIRECT %s %s %s\" - -", req.RemoteAddr, req.Method, rawurl, req.Proto)
				return ctx, &http.Response{
					StatusCode: http.StatusFound,
					Header: http.Header{
						"Location": []string{rawurl},
					},
					Request:       req,
					Close:         true,
					ContentLength: -1,
				}, nil
			}
		}

		if req.URL.Scheme != "http" && !f.shouldForceGAE(req) {
			tr = f.DirectTransport
			if s := req.Header.Get("Connection"); s != "" {
				if s1 := strings.ToLower(s); s != s1 {
					req.Header.Set("Connection", s1)
				}
			}
		}
	}

	if tr != f.DirectTransport && req.Method == http.MethodOptions {
		if v, ok := f.FakeOptionsMatcher.Lookup(req.Host); ok {
			resp := &http.Response{
				StatusCode:    http.StatusOK,
				Header:        http.Header{},
				Request:       req,
				Close:         false,
				ContentLength: -1,
			}
			for _, s := range v.([]string) {
				parts := strings.SplitN(s, ":", 2)
				if len(parts) == 2 {
					resp.Header.Add(parts[0], strings.TrimSpace(parts[1]))
				}
			}
			if origin := req.Header.Get("Origin"); origin != "" {
				resp.Header.Set("Access-Control-Allow-Origin", origin)
			}
			if headers := req.Header.Get("Access-Control-Request-Headers"); headers != "" {
				resp.Header.Set("Access-Control-Allow-Headers", headers)
			}
			glog.V(2).Infof("%s \"GAE FAKEOPTIONS %s %s %s\" %d %s", req.RemoteAddr, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
			return ctx, resp, nil
		}
	}

	prefix := "FETCH"
	if tr == f.DirectTransport {
		prefix = "DIRECT"
	}

	resp, err := tr.RoundTrip(req)
	if err != nil {
		glog.Warningf("%s \"GAE %s %s %s %s\" error: %T(%v)", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, err, err)
		if tr == f.DirectTransport {
			if ne, ok := err.(interface {
				Timeout() bool
			}); ok && ne.Timeout() {
				// f.MultiDialer.ClearCache()
				helpers.TryCloseConnections(tr)
			}
		}
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return ctx, nil, err
	}

	if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/") &&
		resp.Header.Get("Content-Encoding") == "" &&
		f.ForceDeflateMatcher.Match(req.Host) {
		buf := make([]byte, 1024)
		n, err := resp.Body.Read(buf)
		if err != nil {
			defer resp.Body.Close()
			return ctx, nil, err
		}
		buf = buf[:n]
		switch {
		case helpers.IsGzip(buf):
			resp.Header.Set("Content-Encoding", "gzip")
		case helpers.IsBinary(buf):
			resp.Header.Set("Content-Encoding", "deflate")
		}
		resp.Body = helpers.NewMultiReadCloser(bytes.NewReader(buf), resp.Body)
	}

	if resp != nil && resp.Header != nil {
		resp.Header.Del("Alt-Svc")
		resp.Header.Del("Alternate-Protocol")
	}

	glog.V(2).Infof("%s \"GAE %s %s %s %s\" %d %s", req.RemoteAddr, prefix, req.Method, req.URL.String(), req.Proto, resp.StatusCode, resp.Header.Get("Content-Length"))
	return ctx, resp, err
}