func newRequest(hr *http.Request, hc http.ResponseWriter) *Request {
remoteAddrIP, remotePort := hr.RemoteAddr, 0
remoteAddr, _ := net.ResolveTCPAddr("tcp", hr.RemoteAddr)
if remoteAddr != nil {
remoteAddrIP = remoteAddr.IP.String()
remotePort = remoteAddr.Port
}
req := Request{
Method: hr.Method,
URL: hr.URL,
Proto: hr.Proto,
ProtoMajor: hr.ProtoMajor,
ProtoMinor: hr.ProtoMinor,
Headers: hr.Header,
Body: hr.Body,
Close: hr.Close,
Host: hr.Host,
Referer: hr.Referer(),
UserAgent: hr.UserAgent(),
FullParams: hr.Form,
Cookie: hr.Cookies(),
RemoteAddr: remoteAddrIP,
RemotePort: remotePort,
}
return &req
}
// sent a request off to twitter. Returns the response's body or an error.
func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) {
req := new(http.Request)
req.Method = method
req.RawURL = url
req.Host = URLHost
req.Referer = "none"
req.UserAgent = HTTPUserAgent
req.Form = form
req.Header = map[string]string{
"Connection": "Keep Alive",
"Authorization": getAuthHeader(client),
}
req.Body = strings.NewReader(body)
req.URL, err = http.ParseURL(req.RawURL)
if err != nil {
return "", err
}
// send request
resp := new(http.Response)
resp, err = http.Send(req)
if err != nil {
return "", err
}
result = getResponseBody(resp)
return result, nil
}
func main() {
url, err := http.ParseURL("http://bbs.golang-china.org/")
if err != nil {
log.Exit(err)
}
tcpConn, err := net.Dial("tcp", "", url.Host+":80")
if err != nil {
log.Exit(err)
}
clientConn := http.NewClientConn(tcpConn, nil)
var req http.Request
req.URL = url
req.Method = "GET"
req.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.11 (KHTML, like Gecko) Chrome/9.0.570.0 Safari/534.11"
req.Header = map[string]string{}
req.Header["Connection"] = "keep-alive"
err = clientConn.Write(&req)
if err != nil {
log.Exit(err)
}
resp, err := clientConn.Read()
if err != nil {
log.Exit(err)
}
defer resp.Body.Close()
log.Println("Http Response: " + resp.Status)
body, _ := ioutil.ReadAll(resp.Body)
log.Println(string(body))
}
// Downloads url and returns whatever result was.
// This function WILL NOT follow redirects.
func (w *Worker) Download(url *http.URL) (result *FetchResult) {
w.cl_lk.Lock()
client := w.clients[url.Host]
is_new_client := client == nil
if client == nil {
client = new(Client)
client.IOTimeout = w.IOTimeout
w.clients[url.Host] = client
}
w.cl_lk.Unlock()
req := new(http.Request)
req.URL = url
req.Header = make(map[string]string, 10)
req.UserAgent = "HeroshiBot/0.3 (+http://temoto.github.com/heroshi/; [email protected])"
result = client.FetchWithTimeout(req, w.FetchTimeout)
if is_new_client {
go w.staleClient(url.Host, w.KeepAlive)
}
return result
}
// RequestFromMap creates an http.Request from CGI variables.
// The returned Request's Body field is not populated.
func RequestFromMap(params map[string]string) (*http.Request, os.Error) {
r := new(http.Request)
r.Method = params["REQUEST_METHOD"]
if r.Method == "" {
return nil, os.NewError("cgi: no REQUEST_METHOD in environment")
}
r.Proto = params["SERVER_PROTOCOL"]
var ok bool
r.ProtoMajor, r.ProtoMinor, ok = http.ParseHTTPVersion(r.Proto)
if !ok {
return nil, os.NewError("cgi: invalid SERVER_PROTOCOL version")
}
r.Close = true
r.Trailer = http.Header{}
r.Header = http.Header{}
r.Host = params["HTTP_HOST"]
r.Referer = params["HTTP_REFERER"]
r.UserAgent = params["HTTP_USER_AGENT"]
if lenstr := params["CONTENT_LENGTH"]; lenstr != "" {
clen, err := strconv.Atoi64(lenstr)
if err != nil {
return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr)
}
r.ContentLength = clen
}
if ct := params["CONTENT_TYPE"]; ct != "" {
r.Header.Set("Content-Type", ct)
}
// Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers
for k, v := range params {
if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] {
continue
}
r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v)
}
// TODO: cookies. parsing them isn't exported, though.
if r.Host != "" {
// Hostname is provided, so we can reasonably construct a URL,
// even if we have to assume 'http' for the scheme.
r.RawURL = "http://" + r.Host + params["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
// Fallback logic if we don't have a Host header or the URL
// failed to parse
if r.URL == nil {
r.RawURL = params["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
// There's apparently a de-facto standard for this.
// http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636
if s := params["HTTPS"]; s == "on" || s == "ON" || s == "1" {
r.TLS = &tls.ConnectionState{HandshakeComplete: true}
}
// Request.RemoteAddr has its port set by Go's standard http
// server, so we do here too. We don't have one, though, so we
// use a dummy one.
r.RemoteAddr = net.JoinHostPort(params["REMOTE_ADDR"], "0")
return r, nil
}
// Perform the request and follow up to 10 redirects.
// All cookie setting are collected, the final URL is reported.
func DoAndFollow(req *http.Request, dump io.Writer) (response *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
// TODO: set referrer header on redirects.
// Move User-Agent from Header to Request
if ua := req.Header.Get("User-Agent"); ua != "" {
req.UserAgent = ua
req.Header.Del("User-Agent")
}
info("%s %s", req.Method, req.URL.String())
dumpReq(req, dump)
response, err = http.DefaultClient.Do(req)
if err != nil {
return
}
dumpRes(response, dump)
finalUrl = req.URL.String()
cookies = updateCookies(cookies, response.SetCookie)
req.Cookie = updateCookies(req.Cookie, response.SetCookie)
if !shouldRedirect(response.StatusCode) {
return
}
// Start redirecting to final destination
response.Body.Close()
var base = req.URL
// Following the redirect chain is done with a cleaned/empty GET request.
req.Method = "GET"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Header.Del("Content-Type")
req.Header.Del("Content-Length")
req.Header.Del("Accept-Encoding")
req.Header.Del("Connection")
req.Body = nil
for redirect := 0; redirect < 10; redirect++ {
var url string
if url = response.Header.Get("Location"); url == "" {
fmt.Printf("Header:\n%v", response.Header)
err = os.ErrorString(fmt.Sprintf("%d response missing Location header", response.StatusCode))
return
}
if base == nil {
req.URL, err = http.ParseURL(url)
} else {
req.URL, err = base.ParseURL(url)
}
if err != nil {
return
}
url = req.URL.String()
info("GET %s", url)
dumpReq(req, dump)
if response, err = http.DefaultClient.Do(req); err != nil {
return
}
dumpRes(response, dump)
finalUrl = url
cookies = updateCookies(cookies, response.SetCookie)
req.Cookie = updateCookies(req.Cookie, response.SetCookie)
if !shouldRedirect(response.StatusCode) {
return
}
response.Body.Close()
base = req.URL
}
err = os.ErrorString("Too many redirects.")
return
}
func (frontend *Frontend) ServeHTTP(conn http.ResponseWriter, req *http.Request) {
originalHost := req.Host
// Redirect all requests to the "official" public host if the Host header
// doesn't match.
if !frontend.isValidHost(originalHost) {
conn.Header().Set("Location", frontend.RedirectURL)
conn.WriteHeader(http.StatusMovedPermanently)
conn.Write(frontend.RedirectHTML)
frontend.Log(HTTPS_REDIRECT, http.StatusMovedPermanently, originalHost, req)
return
}
// Return the HTTP 503 error page if we're in maintenance mode.
if frontend.MaintenanceMode {
headers := conn.Header()
headers.Set("Content-Type", "text/html; charset=utf-8")
headers.Set("Content-Length", frontend.Error503Length)
conn.WriteHeader(http.StatusServiceUnavailable)
conn.Write(frontend.Error503)
frontend.Log(HTTPS_MAINTENANCE, http.StatusServiceUnavailable, originalHost, req)
return
}
reqPath := req.URL.Path
// Handle requests for any files exposed within the static directory.
if staticFile, ok := frontend.StaticFiles[reqPath]; ok {
expires := time.SecondsToUTC(time.Seconds() + frontend.StaticMaxAge)
headers := conn.Header()
headers.Set("Expires", expires.Format(http.TimeFormat))
headers.Set("Cache-Control", frontend.StaticCache)
headers.Set("Etag", staticFile.ETag)
if req.Header.Get("If-None-Match") == staticFile.ETag {
conn.WriteHeader(http.StatusNotModified)
frontend.Log(HTTPS_STATIC, http.StatusNotModified, originalHost, req)
return
}
// Special case /.well-known/oauth.json?callback= requests.
if reqPath == "/.well-known/oauth.json" && req.URL.RawQuery != "" {
query, err := http.ParseQuery(req.URL.RawQuery)
if err != nil {
logging.Error("Error parsing oauth.json query string %q: %s",
req.URL.RawQuery, err)
frontend.ServeError400(conn, originalHost, req)
return
}
if callbackList, found := query["callback"]; found {
callback := callbackList[0]
if callback != "" {
respLen := len(callback) + len(staticFile.Content) + 2
headers.Set("Content-Type", "text/javascript")
headers.Set("Content-Length", fmt.Sprintf("%d", respLen))
conn.WriteHeader(http.StatusOK)
conn.Write([]byte(callback))
conn.Write([]byte{'('})
conn.Write(staticFile.Content)
conn.Write([]byte{')'})
frontend.Log(HTTPS_STATIC, http.StatusOK, originalHost, req)
return
}
}
}
headers.Set("Content-Type", staticFile.Mimetype)
headers.Set("Content-Length", staticFile.Size)
conn.WriteHeader(http.StatusOK)
conn.Write(staticFile.Content)
frontend.Log(HTTPS_STATIC, http.StatusOK, originalHost, req)
return
}
if frontend.LiveMode {
// Handle WebSocket requests.
if strings.HasPrefix(reqPath, frontend.WebsocketPrefix) {
websocket.Handler(frontend.getWebSocketHandler()).ServeHTTP(conn, req)
return
}
// Handle long-polling Comet requests.
if strings.HasPrefix(reqPath, frontend.CometPrefix) {
query, err := http.ParseQuery(req.URL.RawQuery)
if err != nil {
logging.Error("Error parsing Comet query string %q: %s",
req.URL.RawQuery, err)
frontend.ServeError400(conn, originalHost, req)
return
}
queryReq, found := query["q"]
if !found {
frontend.ServeError400(conn, originalHost, req)
return
}
response, status := getLiveItems(queryReq[0])
headers := conn.Header()
headers.Set("Content-Type", "application/json")
headers.Set("Content-Length", fmt.Sprintf("%d", len(response)))
conn.WriteHeader(status)
conn.Write(response)
frontend.Log(HTTPS_COMET, status, originalHost, req)
return
}
}
// Open a connection to the upstream server.
upstreamConn, err := net.Dial("tcp", frontend.upstreamAddr)
if err != nil {
logging.Error("Couldn't connect to upstream: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
var clientIP string
var upstream net.Conn
splitPoint := strings.LastIndex(req.RemoteAddr, ":")
if splitPoint == -1 {
clientIP = req.RemoteAddr
} else {
clientIP = req.RemoteAddr[0:splitPoint]
}
if frontend.upstreamTLS {
upstream = tls.Client(upstreamConn, tlsconf.Config)
defer upstream.Close()
} else {
upstream = upstreamConn
}
// Modify the request Host: and User-Agent: headers.
req.Host = frontend.upstreamHost
req.Header.Set(
"User-Agent",
fmt.Sprintf("%s, %s, %s", req.UserAgent(), clientIP, originalHost))
// Send the request to the upstream server.
err = req.Write(upstream)
if err != nil {
logging.Error("Error writing to the upstream server: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
// Parse the response from upstream.
resp, err := http.ReadResponse(bufio.NewReader(upstream), req)
if err != nil {
logging.Error("Error parsing response from upstream: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
defer resp.Body.Close()
// Get the original request header.
headers := conn.Header()
// Set a variable to hold the X-Live header value if present.
var liveLength int
if frontend.LiveMode {
xLive := resp.Header.Get("X-Live")
if xLive != "" {
// If the X-Live header was set, parse it into an int.
liveLength, err = strconv.Atoi(xLive)
if err != nil {
logging.Error("Error converting X-Live header value %q: %s", xLive, err)
frontend.ServeError500(conn, originalHost, req)
return
}
resp.Header.Del("X-Live")
}
}
var body []byte
if liveLength > 0 {
var gzipSet bool
var respBody io.ReadCloser
// Check Content-Encoding to see if upstream sent gzipped content.
if resp.Header.Get("Content-Encoding") == "gzip" {
gzipSet = true
respBody, err = gzip.NewReader(resp.Body)
if err != nil {
logging.Error("Error reading gzipped response from upstream: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
defer respBody.Close()
} else {
respBody = resp.Body
}
// Read the X-Live content from the response body.
liveMessage := make([]byte, liveLength)
n, err := respBody.Read(liveMessage)
if n != liveLength || err != nil {
logging.Error("Error reading X-Live response from upstream: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
// Read the response to send back to the original request.
body, err = ioutil.ReadAll(respBody)
if err != nil {
logging.Error("Error reading non X-Live response from upstream: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
// Re-encode the response if it had been gzipped by upstream.
if gzipSet {
buffer := &bytes.Buffer{}
encoder, err := gzip.NewWriter(buffer)
if err != nil {
logging.Error("Error creating a new gzip Writer: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
n, err = encoder.Write(body)
if n != len(body) || err != nil {
logging.Error("Error writing to the gzip Writer: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
err = encoder.Close()
if err != nil {
logging.Error("Error finalising the write to the gzip Writer: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
body = buffer.Bytes()
}
resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(body)))
liveChannel <- liveMessage
} else {
// Read the full response body.
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
logging.Error("Error reading response from upstream: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
}
// Set the received headers back to the initial connection.
for k, values := range resp.Header {
for _, v := range values {
headers.Add(k, v)
}
}
// Write the response body back to the initial connection.
conn.WriteHeader(resp.StatusCode)
conn.Write(body)
frontend.Log(HTTPS_UPSTREAM, resp.StatusCode, originalHost, req)
}
func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) {
r := new(http.Request)
r.Method = env["REQUEST_METHOD"]
if r.Method == "" {
return nil, os.NewError("cgi: no REQUEST_METHOD in environment")
}
r.Close = true
r.Trailer = http.Header{}
r.Header = http.Header{}
r.Host = env["HTTP_HOST"]
r.Referer = env["HTTP_REFERER"]
r.UserAgent = env["HTTP_USER_AGENT"]
// CGI doesn't allow chunked requests, so these should all be accurate:
r.Proto = "HTTP/1.0"
r.ProtoMajor = 1
r.ProtoMinor = 0
r.TransferEncoding = nil
if lenstr := env["CONTENT_LENGTH"]; lenstr != "" {
clen, err := strconv.Atoi64(lenstr)
if err != nil {
return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr)
}
r.ContentLength = clen
r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, clen))
}
// Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers
for k, v := range env {
if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] {
continue
}
r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v)
}
// TODO: cookies. parsing them isn't exported, though.
if r.Host != "" {
// Hostname is provided, so we can reasonably construct a URL,
// even if we have to assume 'http' for the scheme.
r.RawURL = "http://" + r.Host + env["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
// Fallback logic if we don't have a Host header or the URL
// failed to parse
if r.URL == nil {
r.RawURL = env["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
return r, nil
}
func (h *logHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// Strip port number from address
addr := r.RemoteAddr
if colon := strings.LastIndex(addr, ":"); colon != -1 {
addr = addr[:colon]
}
lr := &logRecord{
time: time.UTC(),
ip: addr,
method: r.Method,
rawpath: r.URL.RawPath,
userAgent: r.UserAgent(),
referer: r.Referer(),
responseStatus: http.StatusOK,
proto: r.Proto,
ResponseWriter: rw,
}
h.handler.ServeHTTP(lr, r)
h.ch <- lr
}