Example #1
0
func newRequest(hr *http.Request, hc http.ResponseWriter) *Request {

	remoteAddrIP, remotePort := hr.RemoteAddr, 0
	remoteAddr, _ := net.ResolveTCPAddr("tcp", hr.RemoteAddr)
	if remoteAddr != nil {
		remoteAddrIP = remoteAddr.IP.String()
		remotePort = remoteAddr.Port
	}

	req := Request{
		Method:     hr.Method,
		URL:        hr.URL,
		Proto:      hr.Proto,
		ProtoMajor: hr.ProtoMajor,
		ProtoMinor: hr.ProtoMinor,
		Headers:    hr.Header,
		Body:       hr.Body,
		Close:      hr.Close,
		Host:       hr.Host,
		Referer:    hr.Referer(),
		UserAgent:  hr.UserAgent(),
		FullParams: hr.Form,
		Cookie:     hr.Cookies(),
		RemoteAddr: remoteAddrIP,
		RemotePort: remotePort,
	}
	return &req
}
// sent a request off to twitter. Returns the response's body or an error.
func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) {
	req := new(http.Request)
	req.Method = method
	req.RawURL = url
	req.Host = URLHost
	req.Referer = "none"
	req.UserAgent = HTTPUserAgent
	req.Form = form
	req.Header = map[string]string{
		"Connection":    "Keep Alive",
		"Authorization": getAuthHeader(client),
	}
	req.Body = strings.NewReader(body)
	req.URL, err = http.ParseURL(req.RawURL)
	if err != nil {
		return "", err
	}

	// send request
	resp := new(http.Response)
	resp, err = http.Send(req)
	if err != nil {
		return "", err
	}
	result = getResponseBody(resp)
	return result, nil
}
Example #3
0
func main() {

	url, err := http.ParseURL("http://bbs.golang-china.org/")

	if err != nil {
		log.Exit(err)
	}

	tcpConn, err := net.Dial("tcp", "", url.Host+":80")

	if err != nil {
		log.Exit(err)
	}

	clientConn := http.NewClientConn(tcpConn, nil)

	var req http.Request
	req.URL = url
	req.Method = "GET"
	req.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.11 (KHTML, like Gecko) Chrome/9.0.570.0 Safari/534.11"
	req.Header = map[string]string{}
	req.Header["Connection"] = "keep-alive"

	err = clientConn.Write(&req)
	if err != nil {
		log.Exit(err)
	}
	resp, err := clientConn.Read()
	if err != nil {
		log.Exit(err)
	}
	defer resp.Body.Close()

	log.Println("Http Response: " + resp.Status)
	body, _ := ioutil.ReadAll(resp.Body)

	log.Println(string(body))
}
Example #4
0
// Downloads url and returns whatever result was.
// This function WILL NOT follow redirects.
func (w *Worker) Download(url *http.URL) (result *FetchResult) {
	w.cl_lk.Lock()
	client := w.clients[url.Host]
	is_new_client := client == nil
	if client == nil {
		client = new(Client)
		client.IOTimeout = w.IOTimeout
		w.clients[url.Host] = client
	}
	w.cl_lk.Unlock()

	req := new(http.Request)
	req.URL = url
	req.Header = make(map[string]string, 10)
	req.UserAgent = "HeroshiBot/0.3 (+http://temoto.github.com/heroshi/; [email protected])"

	result = client.FetchWithTimeout(req, w.FetchTimeout)

	if is_new_client {
		go w.staleClient(url.Host, w.KeepAlive)
	}

	return result
}
Example #5
0
// RequestFromMap creates an http.Request from CGI variables.
// The returned Request's Body field is not populated.
func RequestFromMap(params map[string]string) (*http.Request, os.Error) {
	r := new(http.Request)
	r.Method = params["REQUEST_METHOD"]
	if r.Method == "" {
		return nil, os.NewError("cgi: no REQUEST_METHOD in environment")
	}

	r.Proto = params["SERVER_PROTOCOL"]
	var ok bool
	r.ProtoMajor, r.ProtoMinor, ok = http.ParseHTTPVersion(r.Proto)
	if !ok {
		return nil, os.NewError("cgi: invalid SERVER_PROTOCOL version")
	}

	r.Close = true
	r.Trailer = http.Header{}
	r.Header = http.Header{}

	r.Host = params["HTTP_HOST"]
	r.Referer = params["HTTP_REFERER"]
	r.UserAgent = params["HTTP_USER_AGENT"]

	if lenstr := params["CONTENT_LENGTH"]; lenstr != "" {
		clen, err := strconv.Atoi64(lenstr)
		if err != nil {
			return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr)
		}
		r.ContentLength = clen
	}

	if ct := params["CONTENT_TYPE"]; ct != "" {
		r.Header.Set("Content-Type", ct)
	}

	// Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers
	for k, v := range params {
		if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] {
			continue
		}
		r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v)
	}

	// TODO: cookies.  parsing them isn't exported, though.

	if r.Host != "" {
		// Hostname is provided, so we can reasonably construct a URL,
		// even if we have to assume 'http' for the scheme.
		r.RawURL = "http://" + r.Host + params["REQUEST_URI"]
		url, err := http.ParseURL(r.RawURL)
		if err != nil {
			return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL)
		}
		r.URL = url
	}
	// Fallback logic if we don't have a Host header or the URL
	// failed to parse
	if r.URL == nil {
		r.RawURL = params["REQUEST_URI"]
		url, err := http.ParseURL(r.RawURL)
		if err != nil {
			return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL)
		}
		r.URL = url
	}

	// There's apparently a de-facto standard for this.
	// http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636
	if s := params["HTTPS"]; s == "on" || s == "ON" || s == "1" {
		r.TLS = &tls.ConnectionState{HandshakeComplete: true}
	}

	// Request.RemoteAddr has its port set by Go's standard http
	// server, so we do here too. We don't have one, though, so we
	// use a dummy one.
	r.RemoteAddr = net.JoinHostPort(params["REMOTE_ADDR"], "0")

	return r, nil
}
Example #6
0
File: http.go Project: vdobler/ft
// Perform the request and follow up to 10 redirects.
// All cookie setting are collected, the final URL is reported.
func DoAndFollow(req *http.Request, dump io.Writer) (response *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
	// TODO: set referrer header on redirects.

	// Move User-Agent from Header to Request
	if ua := req.Header.Get("User-Agent"); ua != "" {
		req.UserAgent = ua
		req.Header.Del("User-Agent")
	}

	info("%s %s", req.Method, req.URL.String())
	dumpReq(req, dump)
	response, err = http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	dumpRes(response, dump)

	finalUrl = req.URL.String()
	cookies = updateCookies(cookies, response.SetCookie)
	req.Cookie = updateCookies(req.Cookie, response.SetCookie)

	if !shouldRedirect(response.StatusCode) {
		return
	}

	// Start redirecting to final destination
	response.Body.Close()
	var base = req.URL

	// Following the redirect chain is done with a cleaned/empty GET request.
	req.Method = "GET"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Header.Del("Content-Type")
	req.Header.Del("Content-Length")
	req.Header.Del("Accept-Encoding")
	req.Header.Del("Connection")
	req.Body = nil
	for redirect := 0; redirect < 10; redirect++ {
		var url string

		if url = response.Header.Get("Location"); url == "" {
			fmt.Printf("Header:\n%v", response.Header)
			err = os.ErrorString(fmt.Sprintf("%d response missing Location header", response.StatusCode))
			return
		}
		if base == nil {
			req.URL, err = http.ParseURL(url)
		} else {
			req.URL, err = base.ParseURL(url)
		}
		if err != nil {
			return
		}

		url = req.URL.String()
		info("GET %s", url)
		dumpReq(req, dump)

		if response, err = http.DefaultClient.Do(req); err != nil {
			return
		}

		dumpRes(response, dump)
		finalUrl = url
		cookies = updateCookies(cookies, response.SetCookie)
		req.Cookie = updateCookies(req.Cookie, response.SetCookie)

		if !shouldRedirect(response.StatusCode) {
			return
		}
		response.Body.Close()
		base = req.URL

	}
	err = os.ErrorString("Too many redirects.")
	return
}
Example #7
0
func (frontend *Frontend) ServeHTTP(conn http.ResponseWriter, req *http.Request) {

	originalHost := req.Host

	// Redirect all requests to the "official" public host if the Host header
	// doesn't match.
	if !frontend.isValidHost(originalHost) {
		conn.Header().Set("Location", frontend.RedirectURL)
		conn.WriteHeader(http.StatusMovedPermanently)
		conn.Write(frontend.RedirectHTML)
		frontend.Log(HTTPS_REDIRECT, http.StatusMovedPermanently, originalHost, req)
		return
	}

	// Return the HTTP 503 error page if we're in maintenance mode.
	if frontend.MaintenanceMode {
		headers := conn.Header()
		headers.Set("Content-Type", "text/html; charset=utf-8")
		headers.Set("Content-Length", frontend.Error503Length)
		conn.WriteHeader(http.StatusServiceUnavailable)
		conn.Write(frontend.Error503)
		frontend.Log(HTTPS_MAINTENANCE, http.StatusServiceUnavailable, originalHost, req)
		return
	}

	reqPath := req.URL.Path

	// Handle requests for any files exposed within the static directory.
	if staticFile, ok := frontend.StaticFiles[reqPath]; ok {
		expires := time.SecondsToUTC(time.Seconds() + frontend.StaticMaxAge)
		headers := conn.Header()
		headers.Set("Expires", expires.Format(http.TimeFormat))
		headers.Set("Cache-Control", frontend.StaticCache)
		headers.Set("Etag", staticFile.ETag)
		if req.Header.Get("If-None-Match") == staticFile.ETag {
			conn.WriteHeader(http.StatusNotModified)
			frontend.Log(HTTPS_STATIC, http.StatusNotModified, originalHost, req)
			return
		}
		// Special case /.well-known/oauth.json?callback= requests.
		if reqPath == "/.well-known/oauth.json" && req.URL.RawQuery != "" {
			query, err := http.ParseQuery(req.URL.RawQuery)
			if err != nil {
				logging.Error("Error parsing oauth.json query string %q: %s",
					req.URL.RawQuery, err)
				frontend.ServeError400(conn, originalHost, req)
				return
			}
			if callbackList, found := query["callback"]; found {
				callback := callbackList[0]
				if callback != "" {
					respLen := len(callback) + len(staticFile.Content) + 2
					headers.Set("Content-Type", "text/javascript")
					headers.Set("Content-Length", fmt.Sprintf("%d", respLen))
					conn.WriteHeader(http.StatusOK)
					conn.Write([]byte(callback))
					conn.Write([]byte{'('})
					conn.Write(staticFile.Content)
					conn.Write([]byte{')'})
					frontend.Log(HTTPS_STATIC, http.StatusOK, originalHost, req)
					return
				}
			}
		}
		headers.Set("Content-Type", staticFile.Mimetype)
		headers.Set("Content-Length", staticFile.Size)
		conn.WriteHeader(http.StatusOK)
		conn.Write(staticFile.Content)
		frontend.Log(HTTPS_STATIC, http.StatusOK, originalHost, req)
		return
	}

	if frontend.LiveMode {

		// Handle WebSocket requests.
		if strings.HasPrefix(reqPath, frontend.WebsocketPrefix) {
			websocket.Handler(frontend.getWebSocketHandler()).ServeHTTP(conn, req)
			return
		}

		// Handle long-polling Comet requests.
		if strings.HasPrefix(reqPath, frontend.CometPrefix) {
			query, err := http.ParseQuery(req.URL.RawQuery)
			if err != nil {
				logging.Error("Error parsing Comet query string %q: %s",
					req.URL.RawQuery, err)
				frontend.ServeError400(conn, originalHost, req)
				return
			}
			queryReq, found := query["q"]
			if !found {
				frontend.ServeError400(conn, originalHost, req)
				return
			}
			response, status := getLiveItems(queryReq[0])
			headers := conn.Header()
			headers.Set("Content-Type", "application/json")
			headers.Set("Content-Length", fmt.Sprintf("%d", len(response)))
			conn.WriteHeader(status)
			conn.Write(response)
			frontend.Log(HTTPS_COMET, status, originalHost, req)
			return
		}

	}

	// Open a connection to the upstream server.
	upstreamConn, err := net.Dial("tcp", frontend.upstreamAddr)
	if err != nil {
		logging.Error("Couldn't connect to upstream: %s", err)
		frontend.ServeError502(conn, originalHost, req)
		return
	}

	var clientIP string
	var upstream net.Conn

	splitPoint := strings.LastIndex(req.RemoteAddr, ":")
	if splitPoint == -1 {
		clientIP = req.RemoteAddr
	} else {
		clientIP = req.RemoteAddr[0:splitPoint]
	}

	if frontend.upstreamTLS {
		upstream = tls.Client(upstreamConn, tlsconf.Config)
		defer upstream.Close()
	} else {
		upstream = upstreamConn
	}

	// Modify the request Host: and User-Agent: headers.
	req.Host = frontend.upstreamHost
	req.Header.Set(
		"User-Agent",
		fmt.Sprintf("%s, %s, %s", req.UserAgent(), clientIP, originalHost))

	// Send the request to the upstream server.
	err = req.Write(upstream)
	if err != nil {
		logging.Error("Error writing to the upstream server: %s", err)
		frontend.ServeError502(conn, originalHost, req)
		return
	}

	// Parse the response from upstream.
	resp, err := http.ReadResponse(bufio.NewReader(upstream), req)
	if err != nil {
		logging.Error("Error parsing response from upstream: %s", err)
		frontend.ServeError502(conn, originalHost, req)
		return
	}

	defer resp.Body.Close()

	// Get the original request header.
	headers := conn.Header()

	// Set a variable to hold the X-Live header value if present.
	var liveLength int

	if frontend.LiveMode {
		xLive := resp.Header.Get("X-Live")
		if xLive != "" {
			// If the X-Live header was set, parse it into an int.
			liveLength, err = strconv.Atoi(xLive)
			if err != nil {
				logging.Error("Error converting X-Live header value %q: %s", xLive, err)
				frontend.ServeError500(conn, originalHost, req)
				return
			}
			resp.Header.Del("X-Live")
		}
	}

	var body []byte

	if liveLength > 0 {

		var gzipSet bool
		var respBody io.ReadCloser

		// Check Content-Encoding to see if upstream sent gzipped content.
		if resp.Header.Get("Content-Encoding") == "gzip" {
			gzipSet = true
			respBody, err = gzip.NewReader(resp.Body)
			if err != nil {
				logging.Error("Error reading gzipped response from upstream: %s", err)
				frontend.ServeError500(conn, originalHost, req)
				return
			}
			defer respBody.Close()
		} else {
			respBody = resp.Body
		}

		// Read the X-Live content from the response body.
		liveMessage := make([]byte, liveLength)
		n, err := respBody.Read(liveMessage)
		if n != liveLength || err != nil {
			logging.Error("Error reading X-Live response from upstream: %s", err)
			frontend.ServeError500(conn, originalHost, req)
			return
		}

		// Read the response to send back to the original request.
		body, err = ioutil.ReadAll(respBody)
		if err != nil {
			logging.Error("Error reading non X-Live response from upstream: %s", err)
			frontend.ServeError500(conn, originalHost, req)
			return
		}

		// Re-encode the response if it had been gzipped by upstream.
		if gzipSet {
			buffer := &bytes.Buffer{}
			encoder, err := gzip.NewWriter(buffer)
			if err != nil {
				logging.Error("Error creating a new gzip Writer: %s", err)
				frontend.ServeError500(conn, originalHost, req)
				return
			}
			n, err = encoder.Write(body)
			if n != len(body) || err != nil {
				logging.Error("Error writing to the gzip Writer: %s", err)
				frontend.ServeError500(conn, originalHost, req)
				return
			}
			err = encoder.Close()
			if err != nil {
				logging.Error("Error finalising the write to the gzip Writer: %s", err)
				frontend.ServeError500(conn, originalHost, req)
				return
			}
			body = buffer.Bytes()
		}

		resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(body)))
		liveChannel <- liveMessage

	} else {
		// Read the full response body.
		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			logging.Error("Error reading response from upstream: %s", err)
			frontend.ServeError502(conn, originalHost, req)
			return
		}
	}

	// Set the received headers back to the initial connection.
	for k, values := range resp.Header {
		for _, v := range values {
			headers.Add(k, v)
		}
	}

	// Write the response body back to the initial connection.
	conn.WriteHeader(resp.StatusCode)
	conn.Write(body)
	frontend.Log(HTTPS_UPSTREAM, resp.StatusCode, originalHost, req)

}
Example #8
0
func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) {
	r := new(http.Request)
	r.Method = env["REQUEST_METHOD"]
	if r.Method == "" {
		return nil, os.NewError("cgi: no REQUEST_METHOD in environment")
	}
	r.Close = true
	r.Trailer = http.Header{}
	r.Header = http.Header{}

	r.Host = env["HTTP_HOST"]
	r.Referer = env["HTTP_REFERER"]
	r.UserAgent = env["HTTP_USER_AGENT"]

	// CGI doesn't allow chunked requests, so these should all be accurate:
	r.Proto = "HTTP/1.0"
	r.ProtoMajor = 1
	r.ProtoMinor = 0
	r.TransferEncoding = nil

	if lenstr := env["CONTENT_LENGTH"]; lenstr != "" {
		clen, err := strconv.Atoi64(lenstr)
		if err != nil {
			return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr)
		}
		r.ContentLength = clen
		r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, clen))
	}

	// Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers
	for k, v := range env {
		if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] {
			continue
		}
		r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v)
	}

	// TODO: cookies.  parsing them isn't exported, though.

	if r.Host != "" {
		// Hostname is provided, so we can reasonably construct a URL,
		// even if we have to assume 'http' for the scheme.
		r.RawURL = "http://" + r.Host + env["REQUEST_URI"]
		url, err := http.ParseURL(r.RawURL)
		if err != nil {
			return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL)
		}
		r.URL = url
	}
	// Fallback logic if we don't have a Host header or the URL
	// failed to parse
	if r.URL == nil {
		r.RawURL = env["REQUEST_URI"]
		url, err := http.ParseURL(r.RawURL)
		if err != nil {
			return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL)
		}
		r.URL = url
	}
	return r, nil
}
Example #9
-1
func (h *logHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	// Strip port number from address
	addr := r.RemoteAddr
	if colon := strings.LastIndex(addr, ":"); colon != -1 {
		addr = addr[:colon]
	}

	lr := &logRecord{
		time:           time.UTC(),
		ip:             addr,
		method:         r.Method,
		rawpath:        r.URL.RawPath,
		userAgent:      r.UserAgent(),
		referer:        r.Referer(),
		responseStatus: http.StatusOK,
		proto:          r.Proto,
		ResponseWriter: rw,
	}
	h.handler.ServeHTTP(lr, r)
	h.ch <- lr
}