// Read reads the next response from the wire. A valid response might be
// returned together with an ErrPersistEOF, which means that the remote
// requested that this be the last request serviced. Read can be called
// concurrently with Write, but not with another Read.
func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) {
// Retrieve the pipeline ID of this request/response pair
cc.lk.Lock()
id, ok := cc.pipereq[req]
delete(cc.pipereq, req)
if !ok {
cc.lk.Unlock()
return nil, ErrPipeline
}
cc.lk.Unlock()
// Ensure pipeline order
cc.pipe.StartResponse(id)
defer cc.pipe.EndResponse(id)
cc.lk.Lock()
if cc.re != nil {
defer cc.lk.Unlock()
return nil, cc.re
}
if cc.r == nil { // connection closed by user in the meantime
defer cc.lk.Unlock()
return nil, errClosed
}
r := cc.r
lastbody := cc.lastbody
cc.lastbody = nil
cc.lk.Unlock()
// Make sure body is fully consumed, even if user does not call body.Close
if lastbody != nil {
// body.Close is assumed to be idempotent and multiple calls to
// it should return the error that its first invocation
// returned.
err = lastbody.Close()
if err != nil {
cc.lk.Lock()
defer cc.lk.Unlock()
cc.re = err
return nil, err
}
}
resp, err = http.ReadResponse(r, req)
cc.lk.Lock()
defer cc.lk.Unlock()
if err != nil {
cc.re = err
return resp, err
}
cc.lastbody = resp.Body
cc.nread++
if resp.Close {
cc.re = ErrPersistEOF // don't send any more requests
return resp, cc.re
}
return resp, err
}
// TestCopyError tests that we kill the process if there's an error copying
// its output. (for example, from the client having gone away)
func TestCopyError(t *testing.T) {
check(t)
if runtime.GOOS == "windows" {
t.Skipf("skipping test on %q", runtime.GOOS)
}
h := &Handler{
Path: "testdata/test.cgi",
Root: "/test.cgi",
}
ts := httptest.NewServer(h)
defer ts.Close()
conn, err := net.Dial("tcp", ts.Listener.Addr().String())
if err != nil {
t.Fatal(err)
}
req, _ := http.NewRequest("GET", "http://example.com/test.cgi?bigresponse=1", nil)
err = req.Write(conn)
if err != nil {
t.Fatalf("Write: %v", err)
}
res, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
t.Fatalf("ReadResponse: %v", err)
}
pidstr := res.Header.Get("X-CGI-Pid")
if pidstr == "" {
t.Fatalf("expected an X-CGI-Pid header in response")
}
pid, err := strconv.Atoi(pidstr)
if err != nil {
t.Fatalf("invalid X-CGI-Pid value")
}
var buf [5000]byte
n, err := io.ReadFull(res.Body, buf[:])
if err != nil {
t.Fatalf("ReadFull: %d bytes, %v", n, err)
}
childRunning := func() bool {
return isProcessRunning(t, pid)
}
if !childRunning() {
t.Fatalf("pre-conn.Close, expected child to be running")
}
conn.Close()
tries := 0
for tries < 25 && childRunning() {
time.Sleep(50 * time.Millisecond * time.Duration(tries))
tries++
}
if childRunning() {
t.Fatalf("post-conn.Close, expected child to be gone")
}
}
func readResponse(buf *bufio.Reader) string {
req, err := http.NewRequest("GET", srv.URL, nil)
panicOnErr(err, "NewRequest")
resp, err := http.ReadResponse(buf, req)
panicOnErr(err, "resp.Read")
defer resp.Body.Close()
txt, err := ioutil.ReadAll(resp.Body)
panicOnErr(err, "resp.Read")
return string(txt)
}
func TestSimpleMitm(t *testing.T) {
proxy := goproxy.NewProxyHttpServer()
proxy.OnRequest(goproxy.ReqHostIs(https.Listener.Addr().String())).HandleConnect(goproxy.AlwaysMitm)
proxy.OnRequest(goproxy.ReqHostIs("no such host exists")).HandleConnect(goproxy.AlwaysMitm)
client, l := oneShotProxy(proxy, t)
defer l.Close()
c, err := tls.Dial("tcp", https.Listener.Addr().String(), &tls.Config{InsecureSkipVerify: true})
if err != nil {
t.Fatal("cannot dial to tcp server", err)
}
origCert := getCert(c, t)
c.Close()
c2, err := net.Dial("tcp", l.Listener.Addr().String())
if err != nil {
t.Fatal("dialing to proxy", err)
}
creq, err := http.NewRequest("CONNECT", https.URL, nil)
//creq,err := http.NewRequest("CONNECT","https://google.com:443",nil)
if err != nil {
t.Fatal("create new request", creq)
}
creq.Write(c2)
c2buf := bufio.NewReader(c2)
resp, err := http.ReadResponse(c2buf, creq)
if err != nil || resp.StatusCode != 200 {
t.Fatal("Cannot CONNECT through proxy", err)
}
c2tls := tls.Client(c2, &tls.Config{InsecureSkipVerify: true})
proxyCert := getCert(c2tls, t)
if bytes.Equal(proxyCert, origCert) {
t.Errorf("Certificate after mitm is not different\n%v\n%v",
base64.StdEncoding.EncodeToString(origCert),
base64.StdEncoding.EncodeToString(proxyCert))
}
if resp := string(getOrFail(https.URL+"/bobo", client, t)); resp != "bobo" {
t.Error("Wrong response when mitm", resp, "expected bobo")
}
if resp := string(getOrFail(https.URL+"/query?result=bar", client, t)); resp != "bar" {
t.Error("Wrong response when mitm", resp, "expected bar")
}
}
func (pc *persistConn) readLoop() {
alive := true
var lastbody io.ReadCloser // last response body, if any, read on this connection
for alive {
pb, err := pc.br.Peek(1)
pc.lk.Lock()
if pc.numExpectedResponses == 0 {
pc.closeLocked()
pc.lk.Unlock()
if len(pb) > 0 {
log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v",
string(pb), err)
}
return
}
pc.lk.Unlock()
rc := <-pc.reqch
// Advance past the previous response's body, if the
// caller hasn't done so.
if lastbody != nil {
lastbody.Close() // assumed idempotent
lastbody = nil
}
resp, err := http.ReadResponse(pc.br, rc.req)
if err != nil {
pc.close()
} else {
hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" {
resp.Header.Del("Content-Encoding")
resp.Header.Del("Content-Length")
resp.ContentLength = -1
gzReader, zerr := gzip.NewReader(resp.Body)
if zerr != nil {
pc.close()
err = zerr
} else {
resp.Body = &readFirstCloseBoth{&discardOnCloseReadCloser{gzReader}, resp.Body}
}
}
resp.Body = &bodyEOFSignal{body: resp.Body}
}
if err != nil || resp.Close || rc.req.Close {
alive = false
}
hasBody := resp != nil && resp.ContentLength != 0
var waitForBodyRead chan bool
if alive {
if hasBody {
lastbody = resp.Body
waitForBodyRead = make(chan bool)
resp.Body.(*bodyEOFSignal).fn = func() {
if !pc.t.putIdleConn(pc) {
alive = false
}
waitForBodyRead <- true
}
} else {
// When there's no response body, we immediately
// reuse the TCP connection (putIdleConn), but
// we need to prevent ClientConn.Read from
// closing the Response.Body on the next
// loop, otherwise it might close the body
// before the client code has had a chance to
// read it (even though it'll just be 0, EOF).
lastbody = nil
if !pc.t.putIdleConn(pc) {
alive = false
}
}
}
rc.ch <- responseAndError{resp, err}
// Wait for the just-returned response body to be fully consumed
// before we race and peek on the underlying bufio reader.
if waitForBodyRead != nil {
<-waitForBodyRead
}
}
}
// getConn dials and creates a new persistConn to the target as
// specified in the connectMethod. This includes doing a proxy CONNECT
// and/or setting up TLS. If this doesn't return an error, the persistConn
// is ready to write requests to.
func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) {
if pc := t.getIdleConn(cm); pc != nil {
return pc, nil
}
conn, raddr, ip, err := t.dial("tcp", cm.addr())
if err != nil {
if cm.proxyURL != nil {
err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
}
return nil, err
}
pa := cm.proxyAuth()
pconn := &persistConn{
t: t,
cacheKey: cm.String(),
conn: conn,
reqch: make(chan requestAndChan, 50),
host: raddr,
ip: ip,
}
switch {
case cm.proxyURL == nil:
// Do nothing.
case cm.targetScheme == "http":
pconn.isProxy = true
if pa != "" {
pconn.mutateHeaderFunc = func(h http.Header) {
h.Set("Proxy-Authorization", pa)
}
}
case cm.targetScheme == "https":
connectReq := &http.Request{
Method: "CONNECT",
URL: &url.URL{Opaque: cm.targetAddr},
Host: cm.targetAddr,
Header: make(http.Header),
}
if pa != "" {
connectReq.Header.Set("Proxy-Authorization", pa)
}
connectReq.Write(conn)
// Read response.
// Okay to use and discard buffered reader here, because
// TLS server will not speak until spoken to.
br := bufio.NewReader(conn)
resp, err := http.ReadResponse(br, connectReq)
if err != nil {
conn.Close()
return nil, err
}
if resp.StatusCode != 200 {
f := strings.SplitN(resp.Status, " ", 2)
conn.Close()
return nil, errors.New(f[1])
}
}
if cm.targetScheme == "https" {
// Initiate TLS and check remote host name against certificate.
conn = tls.Client(conn, t.TLSClientConfig)
if err = conn.(*tls.Conn).Handshake(); err != nil {
return nil, err
}
if t.TLSClientConfig == nil || !t.TLSClientConfig.InsecureSkipVerify {
if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil {
return nil, err
}
}
pconn.conn = conn
}
pconn.br = bufio.NewReader(pconn.conn)
pconn.bw = bufio.NewWriter(pconn.conn)
go pconn.readLoop()
return pconn, nil
}
func TestChunkedResponse(t *testing.T) {
l, err := net.Listen("tcp", ":10234")
panicOnErr(err, "listen")
defer l.Close()
go func() {
for i := 0; i < 2; i++ {
c, err := l.Accept()
panicOnErr(err, "accept")
_, err = http.ReadRequest(bufio.NewReader(c))
panicOnErr(err, "readrequest")
io.WriteString(c, "HTTP/1.1 200 OK\r\n"+
"Content-Type: text/plain\r\n"+
"Transfer-Encoding: chunked\r\n\r\n"+
"25\r\n"+
"This is the data in the first chunk\r\n\r\n"+
"1C\r\n"+
"and this is the second one\r\n\r\n"+
"3\r\n"+
"con\r\n"+
"8\r\n"+
"sequence\r\n0\r\n\r\n")
c.Close()
}
}()
c, err := net.Dial("tcp", "localhost:10234")
panicOnErr(err, "dial")
defer c.Close()
req, _ := http.NewRequest("GET", "/", nil)
req.Write(c)
resp, err := http.ReadResponse(bufio.NewReader(c), req)
panicOnErr(err, "readresp")
b, err := ioutil.ReadAll(resp.Body)
panicOnErr(err, "readall")
expected := "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
if string(b) != expected {
t.Errorf("Got `%v` expected `%v`", string(b), expected)
}
proxy := goproxy.NewProxyHttpServer()
proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
panicOnErr(ctx.Error, "error reading output")
b, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
panicOnErr(err, "readall onresp")
if enc := resp.Header.Get("Transfer-Encoding"); enc != "" {
t.Fatal("Chunked response should be received as plaintext", enc)
}
resp.Body = ioutil.NopCloser(bytes.NewBufferString(strings.Replace(string(b), "e", "E", -1)))
return resp
})
client, s := oneShotProxy(proxy, t)
defer s.Close()
resp, err = client.Get("http://localhost:10234/")
panicOnErr(err, "client.Get")
b, err = ioutil.ReadAll(resp.Body)
panicOnErr(err, "readall proxy")
if string(b) != strings.Replace(expected, "e", "E", -1) {
t.Error("expected", expected, "w/ e->E. Got", string(b))
}
}
func main() {
verbose := flag.Bool("v", true, "should every proxy request be logged to stdout")
http_addr := flag.String("httpaddr", ":3129", "proxy http listen address")
https_addr := flag.String("httpsaddr", ":3128", "proxy https listen address")
flag.Parse()
proxy := goproxy.NewProxyHttpServer()
proxy.Verbose = *verbose
if proxy.Verbose {
log.Printf("Server starting up! - configured to listen on http interface %s and https interface %s", *http_addr, *https_addr)
}
proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.Host == "" {
fmt.Fprintln(w, "Cannot handle requests without Host header, e.g., HTTP 1.0")
return
}
req.URL.Scheme = "http"
req.URL.Host = req.Host
proxy.ServeHTTP(w, req)
})
proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))).
HandleConnect(goproxy.AlwaysMitm)
proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*:80$"))).
HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
defer func() {
if e := recover(); e != nil {
ctx.Logf("error connecting to remote: %v", e)
client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
}
client.Close()
}()
clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
remote, err := connectDial(proxy, "tcp", req.URL.Host)
orPanic(err)
remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
for {
req, err := http.ReadRequest(clientBuf.Reader)
orPanic(err)
orPanic(req.Write(remoteBuf))
orPanic(remoteBuf.Flush())
resp, err := http.ReadResponse(remoteBuf.Reader, req)
orPanic(err)
orPanic(resp.Write(clientBuf.Writer))
orPanic(clientBuf.Flush())
}
})
go func() {
log.Fatalln(http.ListenAndServe(*http_addr, proxy))
}()
// listen to the TLS ClientHello but make it a CONNECT request instead
ln, err := net.Listen("tcp", *https_addr)
if err != nil {
log.Fatalf("Error listening for https connections - %v", err)
}
for {
c, err := ln.Accept()
if err != nil {
log.Printf("Error accepting new connection - %v", err)
continue
}
go func(c net.Conn) {
tlsConn, err := vhost.TLS(c)
if err != nil {
log.Printf("Error accepting new connection - %v", err)
}
if tlsConn.Host() == "" {
log.Printf("Cannot support non-SNI enabled clients")
return
}
connectReq := &http.Request{
Method: "CONNECT",
URL: &url.URL{
Opaque: tlsConn.Host(),
Host: net.JoinHostPort(tlsConn.Host(), "443"),
},
Host: tlsConn.Host(),
Header: make(http.Header),
}
resp := dumbResponseWriter{tlsConn}
proxy.ServeHTTP(resp, connectReq)
}(c)
}
}
func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request) {
ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), proxy: proxy}
hij, ok := w.(http.Hijacker)
if !ok {
panic("httpserver does not support hijacking")
}
proxyClient, _, e := hij.Hijack()
if e != nil {
panic("Cannot hijack connection " + e.Error())
}
ctx.Logf("Running %d CONNECT handlers", len(proxy.httpsHandlers))
todo, host := OkConnect, r.URL.Host
for i, h := range proxy.httpsHandlers {
newtodo, newhost := h.HandleConnect(host, ctx)
// If found a result, break the loop immediately
if newtodo != nil {
todo, host = newtodo, newhost
ctx.Logf("on %dth handler: %v %s", i, todo, host)
break
}
}
switch todo.Action {
case ConnectAccept:
if !hasPort.MatchString(host) {
host += ":80"
}
targetSiteCon, err := proxy.connectDial("tcp", host)
if err != nil {
httpError(proxyClient, ctx, err)
return
}
ctx.Logf("Accepting CONNECT to %s", host)
proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n"))
go copyAndClose(ctx, targetSiteCon, proxyClient)
go copyAndClose(ctx, proxyClient, targetSiteCon)
case ConnectHijack:
ctx.Logf("Hijacking CONNECT to %s", host)
proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n"))
todo.Hijack(r, proxyClient, ctx)
case ConnectHTTPMitm:
proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n"))
ctx.Logf("Assuming CONNECT is plain HTTP tunneling, mitm proxying it")
targetSiteCon, err := proxy.connectDial("tcp", host)
if err != nil {
ctx.Warnf("Error dialing to %s: %s", host, err.Error())
return
}
for {
client := bufio.NewReader(proxyClient)
remote := bufio.NewReader(targetSiteCon)
req, err := http.ReadRequest(client)
if err != nil && err != io.EOF {
ctx.Warnf("cannot read request of MITM HTTP client: %+#v", err)
}
if err != nil {
return
}
req, resp := proxy.filterRequest(req, ctx)
if resp == nil {
if err := req.Write(targetSiteCon); err != nil {
httpError(proxyClient, ctx, err)
return
}
resp, err = http.ReadResponse(remote, req)
if err != nil {
httpError(proxyClient, ctx, err)
return
}
}
resp = proxy.filterResponse(resp, ctx)
if err := resp.Write(proxyClient); err != nil {
httpError(proxyClient, ctx, err)
return
}
}
case ConnectMitm:
proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n"))
ctx.Logf("Assuming CONNECT is TLS, mitm proxying it")
// this goes in a separate goroutine, so that the net/http server won't think we're
// still handling the request even after hijacking the connection. Those HTTP CONNECT
// request can take forever, and the server will be stuck when "closed".
// TODO: Allow Server.Close() mechanism to shut down this connection as nicely as possible
tlsConfig := defaultTLSConfig
if todo.TLSConfig != nil {
var err error
tlsConfig, err = todo.TLSConfig(host, ctx)
if err != nil {
httpError(proxyClient, ctx, err)
return
}
}
go func() {
//TODO: cache connections to the remote website
rawClientTls := tls.Server(proxyClient, tlsConfig)
if err := rawClientTls.Handshake(); err != nil {
ctx.Warnf("Cannot handshake client %v %v", r.Host, err)
return
}
defer rawClientTls.Close()
clientTlsReader := bufio.NewReader(rawClientTls)
for !isEof(clientTlsReader) {
req, err := http.ReadRequest(clientTlsReader)
if err != nil && err != io.EOF {
return
}
if err != nil {
ctx.Warnf("Cannot read TLS request from mitm'd client %v %v", r.Host, err)
return
}
req.RemoteAddr = r.RemoteAddr // since we're converting the request, need to carry over the original connecting IP as well
ctx.Logf("req %v", r.Host)
req.URL, err = url.Parse("https://" + r.Host + req.URL.String())
// Bug fix which goproxy fails to provide request
// information URL in the context when does HTTPS MITM
ctx.Req = req
req, resp := proxy.filterRequest(req, ctx)
if resp == nil {
if err != nil {
ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path)
return
}
removeProxyHeaders(ctx, req)
resp, err = ctx.RoundTrip(req)
if err != nil {
ctx.Warnf("Cannot read TLS response from mitm'd server %v", err)
return
}
ctx.Logf("resp %v", resp.Status)
}
resp = proxy.filterResponse(resp, ctx)
text := resp.Status
statusCode := strconv.Itoa(resp.StatusCode) + " "
if strings.HasPrefix(text, statusCode) {
text = text[len(statusCode):]
}
// always use 1.1 to support chunked encoding
if _, err := io.WriteString(rawClientTls, "HTTP/1.1"+" "+statusCode+text+"\r\n"); err != nil {
ctx.Warnf("Cannot write TLS response HTTP status from mitm'd client: %v", err)
return
}
// Since we don't know the length of resp, return chunked encoded response
// TODO: use a more reasonable scheme
resp.Header.Del("Content-Length")
resp.Header.Set("Transfer-Encoding", "chunked")
if err := resp.Header.Write(rawClientTls); err != nil {
ctx.Warnf("Cannot write TLS response header from mitm'd client: %v", err)
return
}
if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil {
ctx.Warnf("Cannot write TLS response header end from mitm'd client: %v", err)
return
}
chunked := newChunkedWriter(rawClientTls)
if _, err := io.Copy(chunked, resp.Body); err != nil {
ctx.Warnf("Cannot write TLS response body from mitm'd client: %v", err)
return
}
if err := chunked.Close(); err != nil {
ctx.Warnf("Cannot write TLS chunked EOF from mitm'd client: %v", err)
return
}
if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil {
ctx.Warnf("Cannot write TLS response chunked trailer from mitm'd client: %v", err)
return
}
}
ctx.Logf("Exiting on EOF")
}()
case ConnectReject:
if ctx.Resp != nil {
if err := ctx.Resp.Write(proxyClient); err != nil {
ctx.Warnf("Cannot write response that reject http CONNECT: %v", err)
}
}
proxyClient.Close()
}
}
func (proxy *ProxyHttpServer) NewConnectDialToProxy(https_proxy string) func(network, addr string) (net.Conn, error) {
u, err := url.Parse(https_proxy)
if err != nil {
return nil
}
if u.Scheme == "" || u.Scheme == "http" {
if strings.IndexRune(u.Host, ':') == -1 {
u.Host += ":80"
}
return func(network, addr string) (net.Conn, error) {
connectReq := &http.Request{
Method: "CONNECT",
URL: &url.URL{Opaque: addr},
Host: addr,
Header: make(http.Header),
}
c, err := proxy.dial(network, u.Host)
if err != nil {
return nil, err
}
connectReq.Write(c)
// Read response.
// Okay to use and discard buffered reader here, because
// TLS server will not speak until spoken to.
br := bufio.NewReader(c)
resp, err := http.ReadResponse(br, connectReq)
if err != nil {
c.Close()
return nil, err
}
if resp.StatusCode != 200 {
resp, _ := ioutil.ReadAll(resp.Body)
c.Close()
return nil, errors.New("proxy refused connection" + string(resp))
}
return c, nil
}
}
if u.Scheme == "https" {
if strings.IndexRune(u.Host, ':') == -1 {
u.Host += ":443"
}
return func(network, addr string) (net.Conn, error) {
c, err := proxy.dial(network, u.Host)
if err != nil {
return nil, err
}
c = tls.Client(c, proxy.Tr.TLSClientConfig)
connectReq := &http.Request{
Method: "CONNECT",
URL: &url.URL{Opaque: addr},
Host: addr,
Header: make(http.Header),
}
connectReq.Write(c)
// Read response.
// Okay to use and discard buffered reader here, because
// TLS server will not speak until spoken to.
br := bufio.NewReader(c)
resp, err := http.ReadResponse(br, connectReq)
if err != nil {
c.Close()
return nil, err
}
if resp.StatusCode != 200 {
body, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 500))
resp.Body.Close()
c.Close()
return nil, errors.New("proxy refused connection" + string(body))
}
return c, nil
}
}
return nil
}
