func (t *logTransport) RoundTrip(req *http.Request) (*http.Response, os.Error) { var buf bytes.Buffer os.Stdout.Write([]byte("\n[request]\n")) if req.Body != nil { req.Body = ioutil.NopCloser(&readButCopy{req.Body, &buf}) } req.Write(os.Stdout) if req.Body != nil { req.Body = ioutil.NopCloser(&buf) } os.Stdout.Write([]byte("\n[/request]\n")) res, err := t.rt.RoundTrip(req) fmt.Printf("[response]\n") if err != nil { fmt.Printf("ERROR: %v", err) } else { body := res.Body res.Body = nil res.Write(os.Stdout) if body != nil { res.Body = ioutil.NopCloser(&echoAsRead{body}) } } return res, err }
func (conn *streamConn) connect() (*http.Response, os.Error) { if conn.stale { return nil, os.NewError("Stale connection") } tcpConn, err := net.Dial("tcp", "", conn.url.Host+":80") if err != nil { return nil, err } conn.clientConn = http.NewClientConn(tcpConn, nil) var req http.Request req.URL = conn.url req.Method = "GET" req.Header = map[string]string{} req.Header["Authorization"] = "Basic " + conn.authData if conn.postData != "" { req.Method = "POST" req.Body = nopCloser{bytes.NewBufferString(conn.postData)} req.ContentLength = int64(len(conn.postData)) req.Header["Content-Type"] = "application/x-www-form-urlencoded" } err = conn.clientConn.Write(&req) if err != nil { return nil, err } resp, err := conn.clientConn.Read() if err != nil { return nil, err } return resp, nil }
func NewRequest(method string, url string, doc IDocument) *http.Request { var req http.Request req.Method = method req.ProtoMajor = 1 req.ProtoMinor = 1 req.Close = true req.Header = map[string]string{ "Content-Type": "application/json", "X-Riak-ClientId": "riak.go", } if doc.VectorClock() != "" { req.Header["X-Riak-Vclock"] = doc.VectorClock() } req.TransferEncoding = []string{"chunked"} req.URL, _ = http.ParseURL(url) if doc.Json() != "" { cb := &ClosingBuffer{bytes.NewBufferString(doc.Json())} var rc io.ReadCloser rc = cb req.Body = rc } fmt.Println(req.URL) return &req }
// sent a request off to twitter. Returns the response's body or an error. func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) { req := new(http.Request) req.Method = method req.RawURL = url req.Host = URLHost req.Referer = "none" req.UserAgent = HTTPUserAgent req.Form = form req.Header = map[string]string{ "Connection": "Keep Alive", "Authorization": getAuthHeader(client), } req.Body = strings.NewReader(body) req.URL, err = http.ParseURL(req.RawURL) if err != nil { return "", err } // send request resp := new(http.Response) resp, err = http.Send(req) if err != nil { return "", err } result = getResponseBody(resp) return result, nil }
// PUT /resource/id func (client *Client) Update(id string, body string) (*http.Response, os.Error) { var request *http.Request var err os.Error if request, err = client.newRequest("PUT", id); err != nil { return nil, err } request.Body = nopCloser{bytes.NewBufferString(body)} return client.Request(request) }
func (c *Consumer) httpExecute( method string, url string, body string, oauthParams *OrderedParams) (*http.Response, os.Error) { if c.debug { fmt.Println("httpExecute(method: " + method + ", url: " + url) } var req http.Request req.Method = method req.Header = http.Header{} req.Body = newStringReadCloser(body) parsedurl, err := http.ParseURL(url) if err != nil { return nil, os.NewError("ParseUrl: " + err.String()) } req.URL = parsedurl oauthHdr := "OAuth " for pos, key := range oauthParams.Keys() { if pos > 0 { oauthHdr += "," } oauthHdr += key + "=\"" + oauthParams.Get(key) + "\"" } if c.debug { fmt.Println("AUTH-HDR: " + oauthHdr) } req.Header.Add("Authorization", oauthHdr) resp, err := c.HttpClient.Do(&req) if err != nil { return nil, os.NewError("Do: " + err.String()) } debugHeader := "" for k, vals := range req.Header { for _, val := range vals { debugHeader += "[key: " + k + ", val: " + val + "]" } } if resp.StatusCode != http.StatusOK { bytes, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() return nil, os.NewError("HTTP response is not 200/OK as expected. Actual response: \n" + "\tResponse Status: '" + resp.Status + "'\n" + "\tResponse Code: " + strconv.Itoa(resp.StatusCode) + "\n" + "\tResponse Body: " + string(bytes) + "\n" + "\tRequst Headers: " + debugHeader) } return resp, err }
func sendBody(s Sender, url, method, bodyType string, body io.Reader) (r *http.Response, err os.Error) { var req http.Request req.Method = method req.RawURL = url req.Body = nopCloser{body} req.Header = map[string]string{ "Content-Type": bodyType, } req.TransferEncoding = []string{"chunked"} return Send(s, &req) }
// This function is launched async as a go routine. It tries to send a message // to a remote server and sends a bool to nextChannel to indicate whether this // has succeeded or not. It is not allowed to run this function multiple times in parallel // for the same FederationGateway. func (self *FederationGateway) sendFromQueue(req *WaveletUpdateRequest) { // No HTTP connection open yet? if self.connection == nil { con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", self.manifest.HostName, self.manifest.Port)) if err != nil { // TODO: Good error handling log.Println("Failed connecting to ", self.manifest, err) self.nextChannel <- false return } self.connection = http.NewClientConn(con, nil) } // Build the HTTP request var hreq http.Request hreq.Host = self.manifest.Domain hreq.Header = make(map[string]string) hreq.RawURL = fmt.Sprintf("http://%v:%v/fed%v", self.manifest.HostName, self.manifest.Port, req.uri) hreq.Method = "PUT" hreq.Body = NewRequestBody(req.Data) hreq.ContentLength = int64(len(req.Data)) hreq.Header["Content-Type"] = req.MimeType log.Println("Sending WaveletUpdate to remote server ", hreq.RawURL) // Send the HTTP request self.connection.Write(&hreq) // Read the HTTP response hres, err := self.connection.Read() if err != nil { log.Println("Error reading HTTP response from ", self.manifest, err) // TODO: Better error handling self.connection.Close() self.connection = nil self.nextChannel <- false return } log.Println("After sending WaveletUpdate, status code is ", hres.StatusCode) // Success in sending the wavelet update? if hres.StatusCode == 200 { self.nextChannel <- true return } // Sending the wavelet update failed self.nextChannel <- false }
func nmcPOST(body *bytes.Buffer) (response *http.Response, err os.Error) { client := http.Client{} var req http.Request req.Method = "POST" req.ProtoMajor = 1 req.ProtoMinor = 1 req.Close = true req.Body = ioutil.NopCloser(body) req.Header = http.Header{ "Content-Type": []string{"text/plain"}, "Content-Length": []string{strconv.Itoa(body.Len())}, } req.ContentLength = int64(body.Len()) req.URL = options.nmcURL return client.Do(&req) }
// reads the request body and replaces the buffer with self // returns nil if the body is multipart and not replaced func ReadRequestBody(r *http.Request) (sb *StringBody, err os.Error) { ct := r.Header.Get("Content-Type") // leave it on the buffer if we're multipart if strings.SplitN(ct, ";", 2)[0] != "multipart/form-data" && r.ContentLength > 0 { sb = new(StringBody) const maxFormSize = int64(10 << 20) // 10 MB is a lot of text. b, e := ioutil.ReadAll(io.LimitReader(r.Body, maxFormSize+1)) if e != nil { return nil, e } sb.BodyString = string(b) sb.Close() // to create our buffer r.Body.Close() r.Body = sb return sb, nil } return nil, nil // ignore }
// post taken from Golang modified to allow Headers to be pased func post(url string, headers map[string]string, body io.Reader) (r *http.Response, err os.Error) { var req http.Request req.Method = "POST" req.ProtoMajor = 1 req.ProtoMinor = 1 req.Close = true req.Body = nopCloser{body} for k, v := range headers { req.Header.Add(k, v) } req.TransferEncoding = []string{"chunked"} req.URL, err = http.ParseURL(url) if err != nil { return nil, err } return send(&req) }
// Post issues a POST to the specified URL. // // Caller should close r.Body when done reading it. func Post(url, user, pwd, bodyType string, body io.Reader) (r *http.Response, err os.Error) { var req http.Request req.Method = "POST" req.Body = nopCloser{body} req.Header = map[string]string{ "Content-Type": bodyType, "Transfer-Encoding": "chunked", "X-Twitter-Client": "gotweet", "X-Twitter-Version": "0.1", "Authorization": "Basic " + encodedUsernameAndPassword(user, pwd), } req.URL, err = http.ParseURL(url) if err != nil { return nil, err } return send(&req) }
func (conn *streamConn) connect() (*http.Response, os.Error) { if conn.stale { return nil, os.NewError("Stale connection") } var tcpConn net.Conn var err os.Error if proxy := os.Getenv("HTTP_PROXY"); len(proxy) > 0 { proxy_url, _ := url.Parse(proxy) tcpConn, err = net.Dial("tcp", proxy_url.Host) } else { tcpConn, err = net.Dial("tcp", conn.url.Host+":443") } if err != nil { return nil, err } cf := &tls.Config{Rand: rand.Reader, Time: time.Nanoseconds} ssl := tls.Client(tcpConn, cf) conn.clientConn = http.NewClientConn(ssl, nil) var req http.Request req.URL = conn.url req.Method = "GET" req.Header = http.Header{} req.Header.Set("Authorization", "Basic "+conn.authData) if conn.postData != "" { req.Method = "POST" req.Body = nopCloser{bytes.NewBufferString(conn.postData)} req.ContentLength = int64(len(conn.postData)) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") } resp, err := conn.clientConn.Do(&req) if err != nil { return nil, err } return resp, nil }
// Post issues a POST to the specified URL. // // Caller should close r.Body when done reading it. func authPost(url, user, pwd, client, clientURL, version, agent, bodyType string, body io.Reader) (r *http.Response, err os.Error) { var req http.Request req.Method = "POST" req.Body = body.(io.ReadCloser) req.Header = map[string]string{ "Content-Type": bodyType, "Transfer-Encoding": "chunked", "User-Agent": agent, "X-FluidDB-Client": client, "X-FluidDB-Client-URL": clientURL, "X-FluidDB-Version": version, "Authorization": "Basic " + encodedUsernameAndPassword(user, pwd), } req.URL, err = http.ParseURL(url) if err != nil { return nil, err } return send(&req) }
// Post issues a POST to the specified URL. // // Caller should close r.Body when done reading it. func authPost(url, user, pwd, client, clientURL, version, agent, bodyType string, body io.Reader) (r *http.Response, err os.Error) { var req http.Request req.Method = "POST" req.Body = body.(io.ReadCloser) h := make(http.Header) h.Add("Content-Type", bodyType) h.Add("Transfer-Encoding", "chunked") h.Add("User-Agent", agent) h.Add("X-Twitter-Client", client) h.Add("X-Twitter-Client-URL", clientURL) h.Add("X-Twitter-Version", version) h.Add("Authorization", "Basic "+encodedUsernameAndPassword(user, pwd)) req.Header = h req.URL, err = http.ParseURL(url) if err != nil { return nil, err } return send(&req) }
// PostForm issues a POST to the specified URL, // with data's keys and values urlencoded as the request body. // // Caller should close r.Body when done reading from it. func Post(t *Test) (r *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) { var req http.Request var url = t.Url req.Method = "POST" req.ProtoMajor = 1 req.ProtoMinor = 1 req.Close = true var body *bytes.Buffer var contentType string if hasFile(&t.Param) || t.Method == "POST:mp" { var boundary string body, boundary = multipartBody(&t.Param) contentType = "multipart/form-data; boundary=" + boundary } else { contentType = "application/x-www-form-urlencoded" bodystr := http.EncodeQuery(t.Param) body = bytes.NewBuffer([]byte(bodystr)) } req.Body = nopCloser{body} req.Header = http.Header{ "Content-Type": {contentType}, "Content-Length": {strconv.Itoa(body.Len())}, } addHeadersAndCookies(&req, t) req.ContentLength = int64(body.Len()) req.URL, err = http.ParseURL(url) if err != nil { return nil, url, cookies, err } debug("Will post to %s", req.URL.String()) r, finalUrl, cookies, err = DoAndFollow(&req, t.Dump) return }
func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) { r := new(http.Request) r.Method = env["REQUEST_METHOD"] if r.Method == "" { return nil, os.NewError("cgi: no REQUEST_METHOD in environment") } r.Close = true r.Trailer = http.Header{} r.Header = http.Header{} r.Host = env["HTTP_HOST"] r.Referer = env["HTTP_REFERER"] r.UserAgent = env["HTTP_USER_AGENT"] // CGI doesn't allow chunked requests, so these should all be accurate: r.Proto = "HTTP/1.0" r.ProtoMajor = 1 r.ProtoMinor = 0 r.TransferEncoding = nil if lenstr := env["CONTENT_LENGTH"]; lenstr != "" { clen, err := strconv.Atoi64(lenstr) if err != nil { return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr) } r.ContentLength = clen r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, clen)) } // Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers for k, v := range env { if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] { continue } r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v) } // TODO: cookies. parsing them isn't exported, though. if r.Host != "" { // Hostname is provided, so we can reasonably construct a URL, // even if we have to assume 'http' for the scheme. r.RawURL = "http://" + r.Host + env["REQUEST_URI"] url, err := http.ParseURL(r.RawURL) if err != nil { return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL) } r.URL = url } // Fallback logic if we don't have a Host header or the URL // failed to parse if r.URL == nil { r.RawURL = env["REQUEST_URI"] url, err := http.ParseURL(r.RawURL) if err != nil { return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL) } r.URL = url } return r, nil }
func (s *Server) handleFcgiConnection(fd io.ReadWriteCloser) { br := bufio.NewReader(fd) var req *http.Request var fc *fcgiConn var body bytes.Buffer headers := map[string]string{} for { var h fcgiHeader err := binary.Read(br, binary.BigEndian, &h) if err == os.EOF { break } if err != nil { s.Logger.Println("FCGI Error", err.String()) break } content := make([]byte, h.ContentLength) _, err = io.ReadFull(br, content) if err != nil { s.Logger.Println("FCGI Error", err.String()) break } //read padding if h.PaddingLength > 0 { padding := make([]byte, h.PaddingLength) _, err = io.ReadFull(br, padding) if err != nil { s.Logger.Println("FCGI Error", err.String()) break } } switch h.Type { case fcgiBeginRequest: fc = &fcgiConn{h.RequestId, req, fd, make(map[string][]string), false} case fcgiParams: if h.ContentLength > 0 { readFcgiParams(content, headers) } case fcgiStdin: if h.ContentLength > 0 { body.Write(content) } else if h.ContentLength == 0 { req, _ = cgi.RequestFromMap(headers) req.Body = ioutil.NopCloser(&body) fc.req = req s.routeHandler(req, fc) //we close the connection after processing //TODO: is there a way to keep it open for future requests? fc.complete() return } case fcgiData: if h.ContentLength > 0 { body.Write(content) } case fcgiAbortRequest: } } }
// Perform the request and follow up to 10 redirects. // All cookie setting are collected, the final URL is reported. func DoAndFollow(req *http.Request, dump io.Writer) (response *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) { // TODO: set referrer header on redirects. // Move User-Agent from Header to Request if ua := req.Header.Get("User-Agent"); ua != "" { req.UserAgent = ua req.Header.Del("User-Agent") } info("%s %s", req.Method, req.URL.String()) dumpReq(req, dump) response, err = http.DefaultClient.Do(req) if err != nil { return } dumpRes(response, dump) finalUrl = req.URL.String() cookies = updateCookies(cookies, response.SetCookie) req.Cookie = updateCookies(req.Cookie, response.SetCookie) if !shouldRedirect(response.StatusCode) { return } // Start redirecting to final destination response.Body.Close() var base = req.URL // Following the redirect chain is done with a cleaned/empty GET request. req.Method = "GET" req.ProtoMajor = 1 req.ProtoMinor = 1 req.Header.Del("Content-Type") req.Header.Del("Content-Length") req.Header.Del("Accept-Encoding") req.Header.Del("Connection") req.Body = nil for redirect := 0; redirect < 10; redirect++ { var url string if url = response.Header.Get("Location"); url == "" { fmt.Printf("Header:\n%v", response.Header) err = os.ErrorString(fmt.Sprintf("%d response missing Location header", response.StatusCode)) return } if base == nil { req.URL, err = http.ParseURL(url) } else { req.URL, err = base.ParseURL(url) } if err != nil { return } url = req.URL.String() info("GET %s", url) dumpReq(req, dump) if response, err = http.DefaultClient.Do(req); err != nil { return } dumpRes(response, dump) finalUrl = url cookies = updateCookies(cookies, response.SetCookie) req.Cookie = updateCookies(req.Cookie, response.SetCookie) if !shouldRedirect(response.StatusCode) { return } response.Body.Close() base = req.URL } err = os.ErrorString("Too many redirects.") return }
func (s Session) perform(method, url, data string) { var req http.Request req.URL, _ = http.ParseURL(url) req.Method = method req.Header = s.headers req.ContentLength = int64(len(data)) req.Body = myCloser{bytes.NewBufferString(data)} if *verbose { req.Write(os.Stderr) } retry := 0 request: req.Body = myCloser{bytes.NewBufferString(data)} // recreate anew, in case of retry err := s.conn.Write(&req) if err != nil { if retry < 2 { if err == io.ErrUnexpectedEOF { // the underlying connection has been closed "gracefully" retry++ s.conn.Close() s.conn = dial(s.host) goto request } else if protoerr, ok := err.(*http.ProtocolError); ok && protoerr == http.ErrPersistEOF { // the connection has been closed in an HTTP keepalive sense retry++ s.conn.Close() s.conn = dial(s.host) goto request } } fmt.Fprintln(os.Stderr, "http-gonsole: could not send request:", err) os.Exit(1) } r, err := s.conn.Read(&req) if err != nil { if protoerr, ok := err.(*http.ProtocolError); ok && protoerr == http.ErrPersistEOF { // the remote requested that this be the last request serviced, // we proceed as the response is still valid defer s.conn.Close() defer func() { s.conn = dial(s.host) }() goto output } fmt.Fprintln(os.Stderr, "http-gonsole: could not read response:", err) os.Exit(1) } output: if len(data) > 0 { fmt.Println() } if r.StatusCode >= 500 { fmt.Printf(colorize(C_5xx, "%s %s\n"), r.Proto, r.Status) } else if r.StatusCode >= 400 { fmt.Printf(colorize(C_4xx, "%s %s\n"), r.Proto, r.Status) } else if r.StatusCode >= 300 { fmt.Printf(colorize(C_3xx, "%s %s\n"), r.Proto, r.Status) } else if r.StatusCode >= 200 { fmt.Printf(colorize(C_2xx, "%s %s\n"), r.Proto, r.Status) } if len(r.Header) > 0 { for key, arr := range r.Header { for _, val := range arr { fmt.Printf(colorize(C_Header, "%s: "), key) fmt.Println(val) } } fmt.Println() } if *rememberCookies { if cookies, found := r.Header["Set-Cookie"]; found { for _, h := range cookies { cookie := new(Cookie) cookie.Items = map[string]string{} re, _ := regexp.Compile("^[^=]+=[^;]+(; *(expires=[^;]+|path=[^;,]+|domain=[^;,]+|secure))*,?") rs := re.FindAllString(h, -1) for _, ss := range rs { m := strings.Split(ss, ";", -1) for _, n := range m { t := strings.Split(n, "=", 2) if len(t) == 2 { t[0] = strings.Trim(t[0], " ") t[1] = strings.Trim(t[1], " ") switch t[0] { case "domain": cookie.domain = t[1] case "path": cookie.path = t[1] case "expires": tm, err := time.Parse("Fri, 02-Jan-2006 15:04:05 MST", t[1]) if err != nil { tm, err = time.Parse("Fri, 02-Jan-2006 15:04:05 -0700", t[1]) } cookie.expires = tm case "secure": cookie.secure = true case "HttpOnly": cookie.httpOnly = true default: cookie.Items[t[0]] = t[1] } } } } *s.cookies = append(*s.cookies, cookie) } } } h := r.Header.Get("Content-Length") if len(h) > 0 { n, _ := strconv.Atoi64(h) b := make([]byte, n) io.ReadFull(r.Body, b) fmt.Println(string(b)) } else if method != "HEAD" { b, _ := ioutil.ReadAll(r.Body) fmt.Println(string(b)) } else { // TODO: streaming? } }
func (self *FederationProxy) sendFromQueue() { if len(self.queue) == 0 { return } log.Println("Sending message from queue") // No HTTP connection open yet? if self.connection == nil { con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", self.manifest.HostName, self.manifest.Port)) if err != nil { // TODO: Good error handling log.Println("Failed connecting to ", self.manifest, err) return } self.connection = http.NewClientConn(con, nil) } // Dequeue a message req := self.queue.At(0).(FederationRequest) // Build the HTTP request var hreq http.Request hreq.Host = self.manifest.Domain hreq.Header = make(map[string]string) switch req.(type) { case *PostRequest: preq := req.(*PostRequest) hreq.RawURL = fmt.Sprintf("http://%v:%v/fed/%v", self.manifest.HostName, self.manifest.Port, preq.URI.String()) hreq.Method = "POST" hreq.Body = NewRequestBody(preq.Data) hreq.ContentLength = int64(len(preq.Data)) hreq.Header["Content-Type"] = preq.MimeType case *GetRequest: greq := req.(*GetRequest) hreq.Method = "GET" hreq.RawURL = fmt.Sprintf("http://%v:%v/fed/%v", self.manifest.HostName, self.manifest.Port, greq.URI.String()) default: log.Println(req) panic("Unsupported kind of message forwarded internally to the federation proxy") } log.Println("Sending request to ", hreq.RawURL) // Send the HTTP request self.connection.Write(&hreq) // Read the HTTP response hres, err := self.connection.Read() if err != nil { log.Println("Error reading HTTP response from ", self.manifest, err) // TODO: Better error handling self.connection.Close() self.connection = nil return } // Success. Remove the request from the queue self.queue.Delete(0) // Send the result back to the client _, err = io.Copy(req.GetResponseWriter(), hres.Body) hres.Body.Close() if err != nil { log.Println("Error sending result of federated message back to the client") req.SendFinishSignal(false) } req.SendFinishSignal(true) }
// Put issues a PUT to the specified URL. // // Caller should close r.Body when done reading it. func authPut(url, user, pwd, client, clientURL, version, agent, bodyType string, body io.Reader) (r *http.Response, err os.Error) { var req http.Request req.Method = "PUT" req.Body = body.(io.ReadCloser) if user != "" && pwd != "" { req.Header = map[string][]string{ "Content-Type": {bodyType}, "Transfer-Encoding": {"chunked"}, "User-Agent": {agent}, "X-FluidDB-Client": {client}, "X-FluidDB-Client-URL": {clientURL}, "X-FluidDB-Version": {version}, "Authorization": {"Basic " + encodedUsernameAndPassword(user, pwd)}, } } else { req.Header = map[string][]string{ "Content-Type": {bodyType}, "Transfer-Encoding": {"chunked"}, "User-Agent": {agent}, "X-FluidDB-Client": {client}, "X-FluidDB-Client-URL": {clientURL}, "X-FluidDB-Version": {version}, } } req.URL, err = http.ParseURL(url) if err != nil { return nil, err } return send(&req) }