// sent a request off to twitter. Returns the response's body or an error.
func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) {
req := new(http.Request)
req.Method = method
req.RawURL = url
req.Host = URLHost
req.Referer = "none"
req.UserAgent = HTTPUserAgent
req.Form = form
req.Header = map[string]string{
"Connection": "Keep Alive",
"Authorization": getAuthHeader(client),
}
req.Body = strings.NewReader(body)
req.URL, err = http.ParseURL(req.RawURL)
if err != nil {
return "", err
}
// send request
resp := new(http.Response)
resp, err = http.Send(req)
if err != nil {
return "", err
}
result = getResponseBody(resp)
return result, nil
}
func (proxy *Proxy) ServeHTTP(conn *http.Conn, req *http.Request) {
// Open a connection to the Hub.
hubconn, err := net.Dial("tcp", "", remoteAddr)
if err != nil {
if debugMode {
fmt.Printf("Couldn't connect to remote %s: %v\n", remoteHost, err)
}
return
}
hub := tls.Client(hubconn, tlsconf.Config)
defer hub.Close()
// Modify the request Host: header.
req.Host = remoteHost
// Send the request to the Hub.
err = req.Write(hub)
if err != nil {
if debugMode {
fmt.Printf("Error writing to the hub: %v\n", err)
}
return
}
// Parse the response from the Hub.
resp, err := http.ReadResponse(bufio.NewReader(hub), req.Method)
if err != nil {
if debugMode {
fmt.Printf("Error parsing response from the hub: %v\n", err)
}
return
}
// Set the received headers back to the initial connection.
for k, v := range resp.Header {
conn.SetHeader(k, v)
}
// Read the full response body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
if debugMode {
fmt.Printf("Error reading response from the hub: %v\n", err)
}
resp.Body.Close()
return
}
// Write the response body back to the initial connection.
resp.Body.Close()
conn.WriteHeader(resp.StatusCode)
conn.Write(body)
}
// This function is launched async as a go routine. It tries to send a message
// to a remote server and sends a bool to nextChannel to indicate whether this
// has succeeded or not. It is not allowed to run this function multiple times in parallel
// for the same FederationGateway.
func (self *FederationGateway) sendFromQueue(req *WaveletUpdateRequest) {
// No HTTP connection open yet?
if self.connection == nil {
con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", self.manifest.HostName, self.manifest.Port))
if err != nil {
// TODO: Good error handling
log.Println("Failed connecting to ", self.manifest, err)
self.nextChannel <- false
return
}
self.connection = http.NewClientConn(con, nil)
}
// Build the HTTP request
var hreq http.Request
hreq.Host = self.manifest.Domain
hreq.Header = make(map[string]string)
hreq.RawURL = fmt.Sprintf("http://%v:%v/fed%v", self.manifest.HostName, self.manifest.Port, req.uri)
hreq.Method = "PUT"
hreq.Body = NewRequestBody(req.Data)
hreq.ContentLength = int64(len(req.Data))
hreq.Header["Content-Type"] = req.MimeType
log.Println("Sending WaveletUpdate to remote server ", hreq.RawURL)
// Send the HTTP request
self.connection.Write(&hreq)
// Read the HTTP response
hres, err := self.connection.Read()
if err != nil {
log.Println("Error reading HTTP response from ", self.manifest, err)
// TODO: Better error handling
self.connection.Close()
self.connection = nil
self.nextChannel <- false
return
}
log.Println("After sending WaveletUpdate, status code is ", hres.StatusCode)
// Success in sending the wavelet update?
if hres.StatusCode == 200 {
self.nextChannel <- true
return
}
// Sending the wavelet update failed
self.nextChannel <- false
}
func GetManifest(host string, port uint16) *ServerManifest {
log.Println("Discovery at ", host, port)
// Make a TCP connection
con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", host, port))
if err != nil {
log.Println("Failed connecting to ", host, port, err)
return nil
}
// Make a HTTP connection
hcon := http.NewClientConn(con, nil)
// Build the HTTP request
var hreq http.Request
hreq.Host = host
hreq.Method = "GET"
hreq.RawURL = fmt.Sprintf("http://%v:%v/_manifest", host, port)
log.Println("Sending request")
// Send the HTTP request
if err = hcon.Write(&hreq); err != nil {
log.Println("Error sending GET request")
hcon.Close()
return nil
}
// Read the HTTP response
_, err = hcon.Read()
if err != nil {
log.Println("Error reading HTTP response from ", host, err)
hcon.Close()
return nil
}
m := &ServerManifest{}
// TODO: Parse the manifest
// HACK START
m.HostName = host
m.Port = port
// HACK END
return m
}
func (frontend *Frontend) ServeHTTP(conn *http.Conn, req *http.Request) {
if frontend.enforceHost && req.Host != frontend.officialHost {
conn.SetHeader("Location", frontend.officialRedirectURL)
conn.WriteHeader(http.StatusMovedPermanently)
conn.Write(frontend.officialRedirectHTML)
logRequest(http.StatusMovedPermanently, req.Host, conn)
return
}
originalHost := req.Host
// Open a connection to the App Engine server.
gaeConn, err := net.Dial("tcp", "", frontend.gaeAddr)
if err != nil {
if debugMode {
fmt.Printf("Couldn't connect to remote %s: %v\n", frontend.gaeHost, err)
}
serveError502(conn, originalHost)
return
}
var gae net.Conn
if frontend.gaeTLS {
gae = tls.Client(gaeConn, tlsconf.Config)
defer gae.Close()
} else {
gae = gaeConn
}
// Modify the request Host: header.
req.Host = frontend.gaeHost
// Send the request to the App Engine server.
err = req.Write(gae)
if err != nil {
if debugMode {
fmt.Printf("Error writing to App Engine: %v\n", err)
}
serveError502(conn, originalHost)
return
}
// Parse the response from App Engine.
resp, err := http.ReadResponse(bufio.NewReader(gae), req.Method)
if err != nil {
if debugMode {
fmt.Printf("Error parsing response from App Engine: %v\n", err)
}
serveError502(conn, originalHost)
return
}
// Read the full response body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
if debugMode {
fmt.Printf("Error reading response from App Engine: %v\n", err)
}
serveError502(conn, originalHost)
resp.Body.Close()
return
}
// Set the received headers back to the initial connection.
for k, v := range resp.Header {
conn.SetHeader(k, v)
}
// Write the response body back to the initial connection.
resp.Body.Close()
conn.WriteHeader(resp.StatusCode)
conn.Write(body)
logRequest(resp.StatusCode, originalHost, conn)
}
// RequestFromMap creates an http.Request from CGI variables.
// The returned Request's Body field is not populated.
func RequestFromMap(params map[string]string) (*http.Request, os.Error) {
r := new(http.Request)
r.Method = params["REQUEST_METHOD"]
if r.Method == "" {
return nil, os.NewError("cgi: no REQUEST_METHOD in environment")
}
r.Proto = params["SERVER_PROTOCOL"]
var ok bool
r.ProtoMajor, r.ProtoMinor, ok = http.ParseHTTPVersion(r.Proto)
if !ok {
return nil, os.NewError("cgi: invalid SERVER_PROTOCOL version")
}
r.Close = true
r.Trailer = http.Header{}
r.Header = http.Header{}
r.Host = params["HTTP_HOST"]
r.Referer = params["HTTP_REFERER"]
r.UserAgent = params["HTTP_USER_AGENT"]
if lenstr := params["CONTENT_LENGTH"]; lenstr != "" {
clen, err := strconv.Atoi64(lenstr)
if err != nil {
return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr)
}
r.ContentLength = clen
}
if ct := params["CONTENT_TYPE"]; ct != "" {
r.Header.Set("Content-Type", ct)
}
// Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers
for k, v := range params {
if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] {
continue
}
r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v)
}
// TODO: cookies. parsing them isn't exported, though.
if r.Host != "" {
// Hostname is provided, so we can reasonably construct a URL,
// even if we have to assume 'http' for the scheme.
r.RawURL = "http://" + r.Host + params["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
// Fallback logic if we don't have a Host header or the URL
// failed to parse
if r.URL == nil {
r.RawURL = params["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
// There's apparently a de-facto standard for this.
// http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636
if s := params["HTTPS"]; s == "on" || s == "ON" || s == "1" {
r.TLS = &tls.ConnectionState{HandshakeComplete: true}
}
// Request.RemoteAddr has its port set by Go's standard http
// server, so we do here too. We don't have one, though, so we
// use a dummy one.
r.RemoteAddr = net.JoinHostPort(params["REMOTE_ADDR"], "0")
return r, nil
}
func (frontend *Frontend) ServeHTTP(conn http.ResponseWriter, req *http.Request) {
originalHost := req.Host
// Redirect all requests to the "official" public host if the Host header
// doesn't match.
if !frontend.isValidHost(originalHost) {
conn.Header().Set("Location", frontend.RedirectURL)
conn.WriteHeader(http.StatusMovedPermanently)
conn.Write(frontend.RedirectHTML)
frontend.Log(HTTPS_REDIRECT, http.StatusMovedPermanently, originalHost, req)
return
}
// Return the HTTP 503 error page if we're in maintenance mode.
if frontend.MaintenanceMode {
headers := conn.Header()
headers.Set("Content-Type", "text/html; charset=utf-8")
headers.Set("Content-Length", frontend.Error503Length)
conn.WriteHeader(http.StatusServiceUnavailable)
conn.Write(frontend.Error503)
frontend.Log(HTTPS_MAINTENANCE, http.StatusServiceUnavailable, originalHost, req)
return
}
reqPath := req.URL.Path
// Handle requests for any files exposed within the static directory.
if staticFile, ok := frontend.StaticFiles[reqPath]; ok {
expires := time.SecondsToUTC(time.Seconds() + frontend.StaticMaxAge)
headers := conn.Header()
headers.Set("Expires", expires.Format(http.TimeFormat))
headers.Set("Cache-Control", frontend.StaticCache)
headers.Set("Etag", staticFile.ETag)
if req.Header.Get("If-None-Match") == staticFile.ETag {
conn.WriteHeader(http.StatusNotModified)
frontend.Log(HTTPS_STATIC, http.StatusNotModified, originalHost, req)
return
}
// Special case /.well-known/oauth.json?callback= requests.
if reqPath == "/.well-known/oauth.json" && req.URL.RawQuery != "" {
query, err := http.ParseQuery(req.URL.RawQuery)
if err != nil {
logging.Error("Error parsing oauth.json query string %q: %s",
req.URL.RawQuery, err)
frontend.ServeError400(conn, originalHost, req)
return
}
if callbackList, found := query["callback"]; found {
callback := callbackList[0]
if callback != "" {
respLen := len(callback) + len(staticFile.Content) + 2
headers.Set("Content-Type", "text/javascript")
headers.Set("Content-Length", fmt.Sprintf("%d", respLen))
conn.WriteHeader(http.StatusOK)
conn.Write([]byte(callback))
conn.Write([]byte{'('})
conn.Write(staticFile.Content)
conn.Write([]byte{')'})
frontend.Log(HTTPS_STATIC, http.StatusOK, originalHost, req)
return
}
}
}
headers.Set("Content-Type", staticFile.Mimetype)
headers.Set("Content-Length", staticFile.Size)
conn.WriteHeader(http.StatusOK)
conn.Write(staticFile.Content)
frontend.Log(HTTPS_STATIC, http.StatusOK, originalHost, req)
return
}
if frontend.LiveMode {
// Handle WebSocket requests.
if strings.HasPrefix(reqPath, frontend.WebsocketPrefix) {
websocket.Handler(frontend.getWebSocketHandler()).ServeHTTP(conn, req)
return
}
// Handle long-polling Comet requests.
if strings.HasPrefix(reqPath, frontend.CometPrefix) {
query, err := http.ParseQuery(req.URL.RawQuery)
if err != nil {
logging.Error("Error parsing Comet query string %q: %s",
req.URL.RawQuery, err)
frontend.ServeError400(conn, originalHost, req)
return
}
queryReq, found := query["q"]
if !found {
frontend.ServeError400(conn, originalHost, req)
return
}
response, status := getLiveItems(queryReq[0])
headers := conn.Header()
headers.Set("Content-Type", "application/json")
headers.Set("Content-Length", fmt.Sprintf("%d", len(response)))
conn.WriteHeader(status)
conn.Write(response)
frontend.Log(HTTPS_COMET, status, originalHost, req)
return
}
}
// Open a connection to the upstream server.
upstreamConn, err := net.Dial("tcp", frontend.upstreamAddr)
if err != nil {
logging.Error("Couldn't connect to upstream: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
var clientIP string
var upstream net.Conn
splitPoint := strings.LastIndex(req.RemoteAddr, ":")
if splitPoint == -1 {
clientIP = req.RemoteAddr
} else {
clientIP = req.RemoteAddr[0:splitPoint]
}
if frontend.upstreamTLS {
upstream = tls.Client(upstreamConn, tlsconf.Config)
defer upstream.Close()
} else {
upstream = upstreamConn
}
// Modify the request Host: and User-Agent: headers.
req.Host = frontend.upstreamHost
req.Header.Set(
"User-Agent",
fmt.Sprintf("%s, %s, %s", req.UserAgent(), clientIP, originalHost))
// Send the request to the upstream server.
err = req.Write(upstream)
if err != nil {
logging.Error("Error writing to the upstream server: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
// Parse the response from upstream.
resp, err := http.ReadResponse(bufio.NewReader(upstream), req)
if err != nil {
logging.Error("Error parsing response from upstream: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
defer resp.Body.Close()
// Get the original request header.
headers := conn.Header()
// Set a variable to hold the X-Live header value if present.
var liveLength int
if frontend.LiveMode {
xLive := resp.Header.Get("X-Live")
if xLive != "" {
// If the X-Live header was set, parse it into an int.
liveLength, err = strconv.Atoi(xLive)
if err != nil {
logging.Error("Error converting X-Live header value %q: %s", xLive, err)
frontend.ServeError500(conn, originalHost, req)
return
}
resp.Header.Del("X-Live")
}
}
var body []byte
if liveLength > 0 {
var gzipSet bool
var respBody io.ReadCloser
// Check Content-Encoding to see if upstream sent gzipped content.
if resp.Header.Get("Content-Encoding") == "gzip" {
gzipSet = true
respBody, err = gzip.NewReader(resp.Body)
if err != nil {
logging.Error("Error reading gzipped response from upstream: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
defer respBody.Close()
} else {
respBody = resp.Body
}
// Read the X-Live content from the response body.
liveMessage := make([]byte, liveLength)
n, err := respBody.Read(liveMessage)
if n != liveLength || err != nil {
logging.Error("Error reading X-Live response from upstream: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
// Read the response to send back to the original request.
body, err = ioutil.ReadAll(respBody)
if err != nil {
logging.Error("Error reading non X-Live response from upstream: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
// Re-encode the response if it had been gzipped by upstream.
if gzipSet {
buffer := &bytes.Buffer{}
encoder, err := gzip.NewWriter(buffer)
if err != nil {
logging.Error("Error creating a new gzip Writer: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
n, err = encoder.Write(body)
if n != len(body) || err != nil {
logging.Error("Error writing to the gzip Writer: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
err = encoder.Close()
if err != nil {
logging.Error("Error finalising the write to the gzip Writer: %s", err)
frontend.ServeError500(conn, originalHost, req)
return
}
body = buffer.Bytes()
}
resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(body)))
liveChannel <- liveMessage
} else {
// Read the full response body.
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
logging.Error("Error reading response from upstream: %s", err)
frontend.ServeError502(conn, originalHost, req)
return
}
}
// Set the received headers back to the initial connection.
for k, values := range resp.Header {
for _, v := range values {
headers.Add(k, v)
}
}
// Write the response body back to the initial connection.
conn.WriteHeader(resp.StatusCode)
conn.Write(body)
frontend.Log(HTTPS_UPSTREAM, resp.StatusCode, originalHost, req)
}
func requestFromEnvironment(env map[string]string) (*http.Request, os.Error) {
r := new(http.Request)
r.Method = env["REQUEST_METHOD"]
if r.Method == "" {
return nil, os.NewError("cgi: no REQUEST_METHOD in environment")
}
r.Close = true
r.Trailer = http.Header{}
r.Header = http.Header{}
r.Host = env["HTTP_HOST"]
r.Referer = env["HTTP_REFERER"]
r.UserAgent = env["HTTP_USER_AGENT"]
// CGI doesn't allow chunked requests, so these should all be accurate:
r.Proto = "HTTP/1.0"
r.ProtoMajor = 1
r.ProtoMinor = 0
r.TransferEncoding = nil
if lenstr := env["CONTENT_LENGTH"]; lenstr != "" {
clen, err := strconv.Atoi64(lenstr)
if err != nil {
return nil, os.NewError("cgi: bad CONTENT_LENGTH in environment: " + lenstr)
}
r.ContentLength = clen
r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, clen))
}
// Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers
for k, v := range env {
if !strings.HasPrefix(k, "HTTP_") || skipHeader[k] {
continue
}
r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v)
}
// TODO: cookies. parsing them isn't exported, though.
if r.Host != "" {
// Hostname is provided, so we can reasonably construct a URL,
// even if we have to assume 'http' for the scheme.
r.RawURL = "http://" + r.Host + env["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse host and REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
// Fallback logic if we don't have a Host header or the URL
// failed to parse
if r.URL == nil {
r.RawURL = env["REQUEST_URI"]
url, err := http.ParseURL(r.RawURL)
if err != nil {
return nil, os.NewError("cgi: failed to parse REQUEST_URI into a URL: " + r.RawURL)
}
r.URL = url
}
return r, nil
}
func (self *FederationProxy) sendFromQueue() {
if len(self.queue) == 0 {
return
}
log.Println("Sending message from queue")
// No HTTP connection open yet?
if self.connection == nil {
con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", self.manifest.HostName, self.manifest.Port))
if err != nil {
// TODO: Good error handling
log.Println("Failed connecting to ", self.manifest, err)
return
}
self.connection = http.NewClientConn(con, nil)
}
// Dequeue a message
req := self.queue.At(0).(FederationRequest)
// Build the HTTP request
var hreq http.Request
hreq.Host = self.manifest.Domain
hreq.Header = make(map[string]string)
switch req.(type) {
case *PostRequest:
preq := req.(*PostRequest)
hreq.RawURL = fmt.Sprintf("http://%v:%v/fed/%v", self.manifest.HostName, self.manifest.Port, preq.URI.String())
hreq.Method = "POST"
hreq.Body = NewRequestBody(preq.Data)
hreq.ContentLength = int64(len(preq.Data))
hreq.Header["Content-Type"] = preq.MimeType
case *GetRequest:
greq := req.(*GetRequest)
hreq.Method = "GET"
hreq.RawURL = fmt.Sprintf("http://%v:%v/fed/%v", self.manifest.HostName, self.manifest.Port, greq.URI.String())
default:
log.Println(req)
panic("Unsupported kind of message forwarded internally to the federation proxy")
}
log.Println("Sending request to ", hreq.RawURL)
// Send the HTTP request
self.connection.Write(&hreq)
// Read the HTTP response
hres, err := self.connection.Read()
if err != nil {
log.Println("Error reading HTTP response from ", self.manifest, err)
// TODO: Better error handling
self.connection.Close()
self.connection = nil
return
}
// Success. Remove the request from the queue
self.queue.Delete(0)
// Send the result back to the client
_, err = io.Copy(req.GetResponseWriter(), hres.Body)
hres.Body.Close()
if err != nil {
log.Println("Error sending result of federated message back to the client")
req.SendFinishSignal(false)
}
req.SendFinishSignal(true)
}
