// Read reads the next response from the wire. A valid response might be // returned together with an ErrPersistEOF, which means that the remote // requested that this be the last request serviced. Read can be called // concurrently with Write, but not with another Read. func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { // Retrieve the pipeline ID of this request/response pair cc.lk.Lock() id, ok := cc.pipereq[req] delete(cc.pipereq, req) if !ok { cc.lk.Unlock() return nil, ErrPipeline } cc.lk.Unlock() // Ensure pipeline order cc.pipe.StartResponse(id) defer cc.pipe.EndResponse(id) cc.lk.Lock() if cc.re != nil { defer cc.lk.Unlock() return nil, cc.re } if cc.r == nil { // connection closed by user in the meantime defer cc.lk.Unlock() return nil, errClosed } r := cc.r lastbody := cc.lastbody cc.lastbody = nil cc.lk.Unlock() // Make sure body is fully consumed, even if user does not call body.Close if lastbody != nil { // body.Close is assumed to be idempotent and multiple calls to // it should return the error that its first invocation // returned. err = lastbody.Close() if err != nil { cc.lk.Lock() defer cc.lk.Unlock() cc.re = err return nil, err } } resp, err = http.ReadResponse(r, req) cc.lk.Lock() defer cc.lk.Unlock() if err != nil { cc.re = err return resp, err } cc.lastbody = resp.Body cc.nread++ if resp.Close { cc.re = ErrPersistEOF // don't send any more requests return resp, cc.re } return resp, err }
// TestCopyError tests that we kill the process if there's an error copying // its output. (for example, from the client having gone away) func TestCopyError(t *testing.T) { check(t) if runtime.GOOS == "windows" { t.Skipf("skipping test on %q", runtime.GOOS) } h := &Handler{ Path: "testdata/test.cgi", Root: "/test.cgi", } ts := httptest.NewServer(h) defer ts.Close() conn, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { t.Fatal(err) } req, _ := http.NewRequest("GET", "http://example.com/test.cgi?bigresponse=1", nil) err = req.Write(conn) if err != nil { t.Fatalf("Write: %v", err) } res, err := http.ReadResponse(bufio.NewReader(conn), req) if err != nil { t.Fatalf("ReadResponse: %v", err) } pidstr := res.Header.Get("X-CGI-Pid") if pidstr == "" { t.Fatalf("expected an X-CGI-Pid header in response") } pid, err := strconv.Atoi(pidstr) if err != nil { t.Fatalf("invalid X-CGI-Pid value") } var buf [5000]byte n, err := io.ReadFull(res.Body, buf[:]) if err != nil { t.Fatalf("ReadFull: %d bytes, %v", n, err) } childRunning := func() bool { return isProcessRunning(t, pid) } if !childRunning() { t.Fatalf("pre-conn.Close, expected child to be running") } conn.Close() tries := 0 for tries < 25 && childRunning() { time.Sleep(50 * time.Millisecond * time.Duration(tries)) tries++ } if childRunning() { t.Fatalf("post-conn.Close, expected child to be gone") } }
func readResponse(buf *bufio.Reader) string { req, err := http.NewRequest("GET", srv.URL, nil) panicOnErr(err, "NewRequest") resp, err := http.ReadResponse(buf, req) panicOnErr(err, "resp.Read") defer resp.Body.Close() txt, err := ioutil.ReadAll(resp.Body) panicOnErr(err, "resp.Read") return string(txt) }
func TestSimpleMitm(t *testing.T) { proxy := goproxy.NewProxyHttpServer() proxy.OnRequest(goproxy.ReqHostIs(https.Listener.Addr().String())).HandleConnect(goproxy.AlwaysMitm) proxy.OnRequest(goproxy.ReqHostIs("no such host exists")).HandleConnect(goproxy.AlwaysMitm) client, l := oneShotProxy(proxy, t) defer l.Close() c, err := tls.Dial("tcp", https.Listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) if err != nil { t.Fatal("cannot dial to tcp server", err) } origCert := getCert(c, t) c.Close() c2, err := net.Dial("tcp", l.Listener.Addr().String()) if err != nil { t.Fatal("dialing to proxy", err) } creq, err := http.NewRequest("CONNECT", https.URL, nil) //creq,err := http.NewRequest("CONNECT","https://google.com:443",nil) if err != nil { t.Fatal("create new request", creq) } creq.Write(c2) c2buf := bufio.NewReader(c2) resp, err := http.ReadResponse(c2buf, creq) if err != nil || resp.StatusCode != 200 { t.Fatal("Cannot CONNECT through proxy", err) } c2tls := tls.Client(c2, &tls.Config{InsecureSkipVerify: true}) proxyCert := getCert(c2tls, t) if bytes.Equal(proxyCert, origCert) { t.Errorf("Certificate after mitm is not different\n%v\n%v", base64.StdEncoding.EncodeToString(origCert), base64.StdEncoding.EncodeToString(proxyCert)) } if resp := string(getOrFail(https.URL+"/bobo", client, t)); resp != "bobo" { t.Error("Wrong response when mitm", resp, "expected bobo") } if resp := string(getOrFail(https.URL+"/query?result=bar", client, t)); resp != "bar" { t.Error("Wrong response when mitm", resp, "expected bar") } }
func (pc *persistConn) readLoop() { alive := true var lastbody io.ReadCloser // last response body, if any, read on this connection for alive { pb, err := pc.br.Peek(1) pc.lk.Lock() if pc.numExpectedResponses == 0 { pc.closeLocked() pc.lk.Unlock() if len(pb) > 0 { log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", string(pb), err) } return } pc.lk.Unlock() rc := <-pc.reqch // Advance past the previous response's body, if the // caller hasn't done so. if lastbody != nil { lastbody.Close() // assumed idempotent lastbody = nil } resp, err := http.ReadResponse(pc.br, rc.req) if err != nil { pc.close() } else { hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" { resp.Header.Del("Content-Encoding") resp.Header.Del("Content-Length") resp.ContentLength = -1 gzReader, zerr := gzip.NewReader(resp.Body) if zerr != nil { pc.close() err = zerr } else { resp.Body = &readFirstCloseBoth{&discardOnCloseReadCloser{gzReader}, resp.Body} } } resp.Body = &bodyEOFSignal{body: resp.Body} } if err != nil || resp.Close || rc.req.Close { alive = false } hasBody := resp != nil && resp.ContentLength != 0 var waitForBodyRead chan bool if alive { if hasBody { lastbody = resp.Body waitForBodyRead = make(chan bool) resp.Body.(*bodyEOFSignal).fn = func() { if !pc.t.putIdleConn(pc) { alive = false } waitForBodyRead <- true } } else { // When there's no response body, we immediately // reuse the TCP connection (putIdleConn), but // we need to prevent ClientConn.Read from // closing the Response.Body on the next // loop, otherwise it might close the body // before the client code has had a chance to // read it (even though it'll just be 0, EOF). lastbody = nil if !pc.t.putIdleConn(pc) { alive = false } } } rc.ch <- responseAndError{resp, err} // Wait for the just-returned response body to be fully consumed // before we race and peek on the underlying bufio reader. if waitForBodyRead != nil { <-waitForBodyRead } } }
// getConn dials and creates a new persistConn to the target as // specified in the connectMethod. This includes doing a proxy CONNECT // and/or setting up TLS. If this doesn't return an error, the persistConn // is ready to write requests to. func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) { if pc := t.getIdleConn(cm); pc != nil { return pc, nil } conn, raddr, ip, err := t.dial("tcp", cm.addr()) if err != nil { if cm.proxyURL != nil { err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) } return nil, err } pa := cm.proxyAuth() pconn := &persistConn{ t: t, cacheKey: cm.String(), conn: conn, reqch: make(chan requestAndChan, 50), host: raddr, ip: ip, } switch { case cm.proxyURL == nil: // Do nothing. case cm.targetScheme == "http": pconn.isProxy = true if pa != "" { pconn.mutateHeaderFunc = func(h http.Header) { h.Set("Proxy-Authorization", pa) } } case cm.targetScheme == "https": connectReq := &http.Request{ Method: "CONNECT", URL: &url.URL{Opaque: cm.targetAddr}, Host: cm.targetAddr, Header: make(http.Header), } if pa != "" { connectReq.Header.Set("Proxy-Authorization", pa) } connectReq.Write(conn) // Read response. // Okay to use and discard buffered reader here, because // TLS server will not speak until spoken to. br := bufio.NewReader(conn) resp, err := http.ReadResponse(br, connectReq) if err != nil { conn.Close() return nil, err } if resp.StatusCode != 200 { f := strings.SplitN(resp.Status, " ", 2) conn.Close() return nil, errors.New(f[1]) } } if cm.targetScheme == "https" { // Initiate TLS and check remote host name against certificate. conn = tls.Client(conn, t.TLSClientConfig) if err = conn.(*tls.Conn).Handshake(); err != nil { return nil, err } if t.TLSClientConfig == nil || !t.TLSClientConfig.InsecureSkipVerify { if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil { return nil, err } } pconn.conn = conn } pconn.br = bufio.NewReader(pconn.conn) pconn.bw = bufio.NewWriter(pconn.conn) go pconn.readLoop() return pconn, nil }
func TestChunkedResponse(t *testing.T) { l, err := net.Listen("tcp", ":10234") panicOnErr(err, "listen") defer l.Close() go func() { for i := 0; i < 2; i++ { c, err := l.Accept() panicOnErr(err, "accept") _, err = http.ReadRequest(bufio.NewReader(c)) panicOnErr(err, "readrequest") io.WriteString(c, "HTTP/1.1 200 OK\r\n"+ "Content-Type: text/plain\r\n"+ "Transfer-Encoding: chunked\r\n\r\n"+ "25\r\n"+ "This is the data in the first chunk\r\n\r\n"+ "1C\r\n"+ "and this is the second one\r\n\r\n"+ "3\r\n"+ "con\r\n"+ "8\r\n"+ "sequence\r\n0\r\n\r\n") c.Close() } }() c, err := net.Dial("tcp", "localhost:10234") panicOnErr(err, "dial") defer c.Close() req, _ := http.NewRequest("GET", "/", nil) req.Write(c) resp, err := http.ReadResponse(bufio.NewReader(c), req) panicOnErr(err, "readresp") b, err := ioutil.ReadAll(resp.Body) panicOnErr(err, "readall") expected := "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" if string(b) != expected { t.Errorf("Got `%v` expected `%v`", string(b), expected) } proxy := goproxy.NewProxyHttpServer() proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { panicOnErr(ctx.Error, "error reading output") b, err := ioutil.ReadAll(resp.Body) resp.Body.Close() panicOnErr(err, "readall onresp") if enc := resp.Header.Get("Transfer-Encoding"); enc != "" { t.Fatal("Chunked response should be received as plaintext", enc) } resp.Body = ioutil.NopCloser(bytes.NewBufferString(strings.Replace(string(b), "e", "E", -1))) return resp }) client, s := oneShotProxy(proxy, t) defer s.Close() resp, err = client.Get("http://localhost:10234/") panicOnErr(err, "client.Get") b, err = ioutil.ReadAll(resp.Body) panicOnErr(err, "readall proxy") if string(b) != strings.Replace(expected, "e", "E", -1) { t.Error("expected", expected, "w/ e->E. Got", string(b)) } }
func main() { verbose := flag.Bool("v", true, "should every proxy request be logged to stdout") http_addr := flag.String("httpaddr", ":3129", "proxy http listen address") https_addr := flag.String("httpsaddr", ":3128", "proxy https listen address") flag.Parse() proxy := goproxy.NewProxyHttpServer() proxy.Verbose = *verbose if proxy.Verbose { log.Printf("Server starting up! - configured to listen on http interface %s and https interface %s", *http_addr, *https_addr) } proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Host == "" { fmt.Fprintln(w, "Cannot handle requests without Host header, e.g., HTTP 1.0") return } req.URL.Scheme = "http" req.URL.Host = req.Host proxy.ServeHTTP(w, req) }) proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))). HandleConnect(goproxy.AlwaysMitm) proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*:80$"))). HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) { defer func() { if e := recover(); e != nil { ctx.Logf("error connecting to remote: %v", e) client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n")) } client.Close() }() clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client)) remote, err := connectDial(proxy, "tcp", req.URL.Host) orPanic(err) remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote)) for { req, err := http.ReadRequest(clientBuf.Reader) orPanic(err) orPanic(req.Write(remoteBuf)) orPanic(remoteBuf.Flush()) resp, err := http.ReadResponse(remoteBuf.Reader, req) orPanic(err) orPanic(resp.Write(clientBuf.Writer)) orPanic(clientBuf.Flush()) } }) go func() { log.Fatalln(http.ListenAndServe(*http_addr, proxy)) }() // listen to the TLS ClientHello but make it a CONNECT request instead ln, err := net.Listen("tcp", *https_addr) if err != nil { log.Fatalf("Error listening for https connections - %v", err) } for { c, err := ln.Accept() if err != nil { log.Printf("Error accepting new connection - %v", err) continue } go func(c net.Conn) { tlsConn, err := vhost.TLS(c) if err != nil { log.Printf("Error accepting new connection - %v", err) } if tlsConn.Host() == "" { log.Printf("Cannot support non-SNI enabled clients") return } connectReq := &http.Request{ Method: "CONNECT", URL: &url.URL{ Opaque: tlsConn.Host(), Host: net.JoinHostPort(tlsConn.Host(), "443"), }, Host: tlsConn.Host(), Header: make(http.Header), } resp := dumbResponseWriter{tlsConn} proxy.ServeHTTP(resp, connectReq) }(c) } }
func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request) { ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), proxy: proxy} hij, ok := w.(http.Hijacker) if !ok { panic("httpserver does not support hijacking") } proxyClient, _, e := hij.Hijack() if e != nil { panic("Cannot hijack connection " + e.Error()) } ctx.Logf("Running %d CONNECT handlers", len(proxy.httpsHandlers)) todo, host := OkConnect, r.URL.Host for i, h := range proxy.httpsHandlers { newtodo, newhost := h.HandleConnect(host, ctx) // If found a result, break the loop immediately if newtodo != nil { todo, host = newtodo, newhost ctx.Logf("on %dth handler: %v %s", i, todo, host) break } } switch todo.Action { case ConnectAccept: if !hasPort.MatchString(host) { host += ":80" } targetSiteCon, err := proxy.connectDial("tcp", host) if err != nil { httpError(proxyClient, ctx, err) return } ctx.Logf("Accepting CONNECT to %s", host) proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) go copyAndClose(ctx, targetSiteCon, proxyClient) go copyAndClose(ctx, proxyClient, targetSiteCon) case ConnectHijack: ctx.Logf("Hijacking CONNECT to %s", host) proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) todo.Hijack(r, proxyClient, ctx) case ConnectHTTPMitm: proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) ctx.Logf("Assuming CONNECT is plain HTTP tunneling, mitm proxying it") targetSiteCon, err := proxy.connectDial("tcp", host) if err != nil { ctx.Warnf("Error dialing to %s: %s", host, err.Error()) return } for { client := bufio.NewReader(proxyClient) remote := bufio.NewReader(targetSiteCon) req, err := http.ReadRequest(client) if err != nil && err != io.EOF { ctx.Warnf("cannot read request of MITM HTTP client: %+#v", err) } if err != nil { return } req, resp := proxy.filterRequest(req, ctx) if resp == nil { if err := req.Write(targetSiteCon); err != nil { httpError(proxyClient, ctx, err) return } resp, err = http.ReadResponse(remote, req) if err != nil { httpError(proxyClient, ctx, err) return } } resp = proxy.filterResponse(resp, ctx) if err := resp.Write(proxyClient); err != nil { httpError(proxyClient, ctx, err) return } } case ConnectMitm: proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) ctx.Logf("Assuming CONNECT is TLS, mitm proxying it") // this goes in a separate goroutine, so that the net/http server won't think we're // still handling the request even after hijacking the connection. Those HTTP CONNECT // request can take forever, and the server will be stuck when "closed". // TODO: Allow Server.Close() mechanism to shut down this connection as nicely as possible tlsConfig := defaultTLSConfig if todo.TLSConfig != nil { var err error tlsConfig, err = todo.TLSConfig(host, ctx) if err != nil { httpError(proxyClient, ctx, err) return } } go func() { //TODO: cache connections to the remote website rawClientTls := tls.Server(proxyClient, tlsConfig) if err := rawClientTls.Handshake(); err != nil { ctx.Warnf("Cannot handshake client %v %v", r.Host, err) return } defer rawClientTls.Close() clientTlsReader := bufio.NewReader(rawClientTls) for !isEof(clientTlsReader) { req, err := http.ReadRequest(clientTlsReader) if err != nil && err != io.EOF { return } if err != nil { ctx.Warnf("Cannot read TLS request from mitm'd client %v %v", r.Host, err) return } req.RemoteAddr = r.RemoteAddr // since we're converting the request, need to carry over the original connecting IP as well ctx.Logf("req %v", r.Host) req.URL, err = url.Parse("https://" + r.Host + req.URL.String()) // Bug fix which goproxy fails to provide request // information URL in the context when does HTTPS MITM ctx.Req = req req, resp := proxy.filterRequest(req, ctx) if resp == nil { if err != nil { ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path) return } removeProxyHeaders(ctx, req) resp, err = ctx.RoundTrip(req) if err != nil { ctx.Warnf("Cannot read TLS response from mitm'd server %v", err) return } ctx.Logf("resp %v", resp.Status) } resp = proxy.filterResponse(resp, ctx) text := resp.Status statusCode := strconv.Itoa(resp.StatusCode) + " " if strings.HasPrefix(text, statusCode) { text = text[len(statusCode):] } // always use 1.1 to support chunked encoding if _, err := io.WriteString(rawClientTls, "HTTP/1.1"+" "+statusCode+text+"\r\n"); err != nil { ctx.Warnf("Cannot write TLS response HTTP status from mitm'd client: %v", err) return } // Since we don't know the length of resp, return chunked encoded response // TODO: use a more reasonable scheme resp.Header.Del("Content-Length") resp.Header.Set("Transfer-Encoding", "chunked") if err := resp.Header.Write(rawClientTls); err != nil { ctx.Warnf("Cannot write TLS response header from mitm'd client: %v", err) return } if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil { ctx.Warnf("Cannot write TLS response header end from mitm'd client: %v", err) return } chunked := newChunkedWriter(rawClientTls) if _, err := io.Copy(chunked, resp.Body); err != nil { ctx.Warnf("Cannot write TLS response body from mitm'd client: %v", err) return } if err := chunked.Close(); err != nil { ctx.Warnf("Cannot write TLS chunked EOF from mitm'd client: %v", err) return } if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil { ctx.Warnf("Cannot write TLS response chunked trailer from mitm'd client: %v", err) return } } ctx.Logf("Exiting on EOF") }() case ConnectReject: if ctx.Resp != nil { if err := ctx.Resp.Write(proxyClient); err != nil { ctx.Warnf("Cannot write response that reject http CONNECT: %v", err) } } proxyClient.Close() } }
func (proxy *ProxyHttpServer) NewConnectDialToProxy(https_proxy string) func(network, addr string) (net.Conn, error) { u, err := url.Parse(https_proxy) if err != nil { return nil } if u.Scheme == "" || u.Scheme == "http" { if strings.IndexRune(u.Host, ':') == -1 { u.Host += ":80" } return func(network, addr string) (net.Conn, error) { connectReq := &http.Request{ Method: "CONNECT", URL: &url.URL{Opaque: addr}, Host: addr, Header: make(http.Header), } c, err := proxy.dial(network, u.Host) if err != nil { return nil, err } connectReq.Write(c) // Read response. // Okay to use and discard buffered reader here, because // TLS server will not speak until spoken to. br := bufio.NewReader(c) resp, err := http.ReadResponse(br, connectReq) if err != nil { c.Close() return nil, err } if resp.StatusCode != 200 { resp, _ := ioutil.ReadAll(resp.Body) c.Close() return nil, errors.New("proxy refused connection" + string(resp)) } return c, nil } } if u.Scheme == "https" { if strings.IndexRune(u.Host, ':') == -1 { u.Host += ":443" } return func(network, addr string) (net.Conn, error) { c, err := proxy.dial(network, u.Host) if err != nil { return nil, err } c = tls.Client(c, proxy.Tr.TLSClientConfig) connectReq := &http.Request{ Method: "CONNECT", URL: &url.URL{Opaque: addr}, Host: addr, Header: make(http.Header), } connectReq.Write(c) // Read response. // Okay to use and discard buffered reader here, because // TLS server will not speak until spoken to. br := bufio.NewReader(c) resp, err := http.ReadResponse(br, connectReq) if err != nil { c.Close() return nil, err } if resp.StatusCode != 200 { body, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 500)) resp.Body.Close() c.Close() return nil, errors.New("proxy refused connection" + string(body)) } return c, nil } } return nil }