func TestRequestQueueLess(t *testing.T) {
q := new(requestQueue)
var a, b, c, d http.Request
a.Header = map[string]string{
"X-Pri": "1",
}
b.Header = map[string]string{
"X-Pri": "2",
}
c.Header = map[string]string{
"X-Pri": "1",
}
d.Header = map[string]string{} // default X-Pri: 5000
q.Push(&clientRequest{r: &a})
q.Push(&clientRequest{r: &b})
q.Push(&clientRequest{r: &c})
q.Push(&clientRequest{r: &d})
if !q.Less(0, 1) {
t.Error("want a < b")
}
if q.Less(1, 0) {
t.Error("want b > a")
}
if !q.Less(2, 1) {
t.Error("want c < b")
}
if q.Less(1, 2) {
t.Error("want b > c")
}
if q.Less(0, 2) {
t.Error("want a == c")
}
if q.Less(2, 0) {
t.Error("want c == a")
}
if !q.Less(0, 3) {
t.Error("want a < d")
}
if q.Less(3, 0) {
t.Error("want d > a")
}
}
func (conn *streamConn) connect() (*http.Response, os.Error) {
if conn.stale {
return nil, os.NewError("Stale connection")
}
tcpConn, err := net.Dial("tcp", "", conn.url.Host+":80")
if err != nil {
return nil, err
}
conn.clientConn = http.NewClientConn(tcpConn, nil)
var req http.Request
req.URL = conn.url
req.Method = "GET"
req.Header = map[string]string{}
req.Header["Authorization"] = "Basic " + conn.authData
if conn.postData != "" {
req.Method = "POST"
req.Body = nopCloser{bytes.NewBufferString(conn.postData)}
req.ContentLength = int64(len(conn.postData))
req.Header["Content-Type"] = "application/x-www-form-urlencoded"
}
err = conn.clientConn.Write(&req)
if err != nil {
return nil, err
}
resp, err := conn.clientConn.Read()
if err != nil {
return nil, err
}
return resp, nil
}
func post(url_ string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "POST"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
req.Header = map[string][]string{
"Authorization": {"OAuth "},
}
req.TransferEncoding = []string{"chunked"}
first := true
for k, v := range oauthHeaders {
if first {
first = false
} else {
req.Header["Authorization"][0] += ",\n "
}
req.Header["Authorization"][0] += k + "=\"" + v + "\""
}
req.URL, err = url.Parse(url_)
if err != nil {
return nil, err
}
return send(&req)
}
// Much like http.Get. If s is nil, uses DefaultSender.
func Get(s Sender, url string) (rs []*http.Response, err os.Error) {
for redirect := 0; ; redirect++ {
if redirect >= 10 {
err = os.ErrorString("stopped after 10 redirects")
break
}
var req http.Request
req.RawURL = url
req.Header = map[string]string{}
r, err := Send(s, &req)
if err != nil {
break
}
rs = prepend(r, rs)
if shouldRedirect(r.StatusCode) {
r.Body.Close()
if url = r.GetHeader("Location"); url == "" {
err = os.ErrorString(fmt.Sprintf("%d response missing Location header", r.StatusCode))
break
}
continue
}
return
}
err = &http.URLError{"Get", url, err}
return
}
func NewRequest(method string, url string, doc IDocument) *http.Request {
var req http.Request
req.Method = method
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
req.Header = map[string]string{
"Content-Type": "application/json",
"X-Riak-ClientId": "riak.go",
}
if doc.VectorClock() != "" {
req.Header["X-Riak-Vclock"] = doc.VectorClock()
}
req.TransferEncoding = []string{"chunked"}
req.URL, _ = http.ParseURL(url)
if doc.Json() != "" {
cb := &ClosingBuffer{bytes.NewBufferString(doc.Json())}
var rc io.ReadCloser
rc = cb
req.Body = rc
}
fmt.Println(req.URL)
return &req
}
func Delete(url string) *HttpRequestBuilder {
var req http.Request
req.Method = "DELETE"
req.Header = http.Header{}
req.Header.Set("User-Agent", defaultUserAgent)
return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}
// sent a request off to twitter. Returns the response's body or an error.
func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) {
req := new(http.Request)
req.Method = method
req.RawURL = url
req.Host = URLHost
req.Referer = "none"
req.UserAgent = HTTPUserAgent
req.Form = form
req.Header = map[string]string{
"Connection": "Keep Alive",
"Authorization": getAuthHeader(client),
}
req.Body = strings.NewReader(body)
req.URL, err = http.ParseURL(req.RawURL)
if err != nil {
return "", err
}
// send request
resp := new(http.Response)
resp, err = http.Send(req)
if err != nil {
return "", err
}
result = getResponseBody(resp)
return result, nil
}
func post(theUrl string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
var req http.Request
var authorization string = "OAuth "
req.Method = "POST"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
req.Header = http.Header{}
req.TransferEncoding = []string{"chunked"}
first := true
for k, v := range oauthHeaders {
if first {
first = false
} else {
authorization += ",\n "
}
authorization += k + "=\"" + v + "\""
}
req.Header.Add("Authorization", authorization)
req.URL, err = url.Parse(theUrl)
if err != nil {
return nil, err
}
return send(&req)
}
// Perform a GET request for the test t.
func Get(t *Test) (r *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
var url = t.Url // <-- Patched
var req http.Request
req.Method = "GET"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Header = http.Header{}
if len(t.Param) > 0 {
ep := http.EncodeQuery(t.Param)
if strings.Contains(url, "?") {
url = url + "&" + ep
} else {
url = url + "?" + ep
}
}
req.URL, err = http.ParseURL(url)
if err != nil {
err = &http.URLError{"Get", url, err}
return
}
addHeadersAndCookies(&req, t)
url = req.URL.String()
debug("Will get from %s", req.URL.String())
r, finalUrl, cookies, err = DoAndFollow(&req, t.Dump)
return
}
func (c *Consumer) httpExecute(
method string, url string, body string, oauthParams *OrderedParams) (*http.Response, os.Error) {
if c.debug {
fmt.Println("httpExecute(method: " + method + ", url: " + url)
}
var req http.Request
req.Method = method
req.Header = http.Header{}
req.Body = newStringReadCloser(body)
parsedurl, err := http.ParseURL(url)
if err != nil {
return nil, os.NewError("ParseUrl: " + err.String())
}
req.URL = parsedurl
oauthHdr := "OAuth "
for pos, key := range oauthParams.Keys() {
if pos > 0 {
oauthHdr += ","
}
oauthHdr += key + "=\"" + oauthParams.Get(key) + "\""
}
if c.debug {
fmt.Println("AUTH-HDR: " + oauthHdr)
}
req.Header.Add("Authorization", oauthHdr)
resp, err := c.HttpClient.Do(&req)
if err != nil {
return nil, os.NewError("Do: " + err.String())
}
debugHeader := ""
for k, vals := range req.Header {
for _, val := range vals {
debugHeader += "[key: " + k + ", val: " + val + "]"
}
}
if resp.StatusCode != http.StatusOK {
bytes, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
return nil, os.NewError("HTTP response is not 200/OK as expected. Actual response: \n" +
"\tResponse Status: '" + resp.Status + "'\n" +
"\tResponse Code: " + strconv.Itoa(resp.StatusCode) + "\n" +
"\tResponse Body: " + string(bytes) + "\n" +
"\tRequst Headers: " + debugHeader)
}
return resp, err
}
func sendBody(s Sender, url, method, bodyType string, body io.Reader) (r *http.Response, err os.Error) {
var req http.Request
req.Method = method
req.RawURL = url
req.Body = nopCloser{body}
req.Header = map[string]string{
"Content-Type": bodyType,
}
req.TransferEncoding = []string{"chunked"}
return Send(s, &req)
}
func Send(s Sender, req *http.Request) (resp *http.Response, err os.Error) {
if s == nil {
s = DefaultSender
}
req.ProtoMajor = 1
req.ProtoMinor = 1
header := req.Header
req.Header = map[string]string{}
for k, v := range header {
req.Header[http.CanonicalHeaderKey(k)] = v
}
return s.Send(req)
}
func Get(url, user, pwd string) (r *http.Response, err os.Error) {
var req http.Request
req.Header = map[string]string{"Authorization": "Basic " +
encodedUsernameAndPassword(user, pwd)}
if req.URL, err = http.ParseURL(url); err != nil {
return
}
if r, err = send(&req); err != nil {
return
}
return
}
func authGet(url, user, pwd string) (r *http.Response, err os.Error) {
var req http.Request
h := make(http.Header)
h.Add("Authorization", "Basic "+
encodedUsernameAndPassword(user, pwd))
req.Header = h
if req.URL, err = http.ParseURL(url); err != nil {
return
}
if r, err = send(&req); err != nil {
return
}
return
}
// Head issues a HEAD to the specified URL.
func authHead(url, user, pwd string) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "HEAD"
if user != "" && pwd != "" {
req.Header = map[string][]string{"Authorization": {"Basic " +
encodedUsernameAndPassword(user, pwd)}}
}
if req.URL, err = http.ParseURL(url); err != nil {
return
}
if r, err = send(&req); err != nil {
return
}
return
}
// This function is launched async as a go routine. It tries to send a message
// to a remote server and sends a bool to nextChannel to indicate whether this
// has succeeded or not. It is not allowed to run this function multiple times in parallel
// for the same FederationGateway.
func (self *FederationGateway) sendFromQueue(req *WaveletUpdateRequest) {
// No HTTP connection open yet?
if self.connection == nil {
con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", self.manifest.HostName, self.manifest.Port))
if err != nil {
// TODO: Good error handling
log.Println("Failed connecting to ", self.manifest, err)
self.nextChannel <- false
return
}
self.connection = http.NewClientConn(con, nil)
}
// Build the HTTP request
var hreq http.Request
hreq.Host = self.manifest.Domain
hreq.Header = make(map[string]string)
hreq.RawURL = fmt.Sprintf("http://%v:%v/fed%v", self.manifest.HostName, self.manifest.Port, req.uri)
hreq.Method = "PUT"
hreq.Body = NewRequestBody(req.Data)
hreq.ContentLength = int64(len(req.Data))
hreq.Header["Content-Type"] = req.MimeType
log.Println("Sending WaveletUpdate to remote server ", hreq.RawURL)
// Send the HTTP request
self.connection.Write(&hreq)
// Read the HTTP response
hres, err := self.connection.Read()
if err != nil {
log.Println("Error reading HTTP response from ", self.manifest, err)
// TODO: Better error handling
self.connection.Close()
self.connection = nil
self.nextChannel <- false
return
}
log.Println("After sending WaveletUpdate, status code is ", hres.StatusCode)
// Success in sending the wavelet update?
if hres.StatusCode == 200 {
self.nextChannel <- true
return
}
// Sending the wavelet update failed
self.nextChannel <- false
}
func YadisRequest(url_ string, method string) (resp *http.Response, err os.Error) {
resp = nil
var request = new(http.Request)
var client = new(http.Client)
var Header = make(http.Header)
request.Method = method
request.RawURL = url_
request.URL, err = url.Parse(url_)
if err != nil {
return
}
// Common parameters
request.Proto = "HTTP/1.0"
request.ProtoMajor = 1
request.ProtoMinor = 0
request.ContentLength = 0
request.Close = true
Header.Add("Accept", "application/xrds+xml")
request.Header = Header
// Follow a maximum of 5 redirections
for i := 0; i < 5; i++ {
response, err := client.Do(request)
if err != nil {
return nil, err
}
if response.StatusCode == 301 || response.StatusCode == 302 || response.StatusCode == 303 || response.StatusCode == 307 {
location := response.Header.Get("Location")
request.RawURL = location
request.URL, err = url.Parse(location)
if err != nil {
return
}
} else {
return response, nil
}
}
return nil, os.NewError("Too many redirections")
}
func nmcPOST(body *bytes.Buffer) (response *http.Response, err os.Error) {
client := http.Client{}
var req http.Request
req.Method = "POST"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
req.Body = ioutil.NopCloser(body)
req.Header = http.Header{
"Content-Type": []string{"text/plain"},
"Content-Length": []string{strconv.Itoa(body.Len())},
}
req.ContentLength = int64(body.Len())
req.URL = options.nmcURL
return client.Do(&req)
}
func (client *Client) Fetch(req *http.Request) (result *FetchResult) {
creds := req.URL.RawUserinfo
_, has_auth_header := req.Header["Authorization"]
if len(creds) > 0 && !has_auth_header {
encoded := make([]byte, base64.URLEncoding.EncodedLen(len(creds)))
base64.URLEncoding.Encode(encoded, []byte(creds))
if req.Header == nil {
req.Header = make(map[string]string)
}
req.Header["Authorization"] = "Basic " + string(encoded)
}
// debug
if false {
dump, _ := http.DumpRequest(req, true)
print(string(dump))
}
resp, err := client.Request(req)
if err != nil {
return ErrorResult(req.URL.Raw, err.String())
}
// Prevent simultaneous IO from different goroutines.
client.lk.Lock()
defer client.lk.Unlock()
var buf bytes.Buffer
_, err = io.Copy(&buf, resp.Body)
responseBody := buf.Bytes()
if err != nil {
return ErrorResult(req.URL.Raw, err.String())
}
resp.Body.Close()
return &FetchResult{
Url: req.URL.Raw,
Success: true,
Status: resp.Status,
StatusCode: resp.StatusCode,
Body: responseBody,
Headers: resp.Header,
}
}
func (ds *dummySender) Send(req *http.Request) (*http.Response, os.Error) {
if req.Header == nil {
req.Header = map[string]string{}
}
if etag, ok := req.Header["If-None-Match"]; ok {
if resp, ok := testEtags[etag]; ok {
resp.Status = "304 Not Modified"
resp.StatusCode = 304
return resp, nil
}
}
resp, ok := dummyResponses[req.RawURL]
if !ok {
return nil, os.NewError("no response")
}
resp.Status = "200 OK"
resp.StatusCode = 200
return resp, nil
}
// Post issues a POST to the specified URL.
//
// Caller should close r.Body when done reading it.
func Post(url, user, pwd, bodyType string, body io.Reader) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "POST"
req.Body = nopCloser{body}
req.Header = map[string]string{
"Content-Type": bodyType,
"Transfer-Encoding": "chunked",
"X-Twitter-Client": "gotweet",
"X-Twitter-Version": "0.1",
"Authorization": "Basic " + encodedUsernameAndPassword(user, pwd),
}
req.URL, err = http.ParseURL(url)
if err != nil {
return nil, err
}
return send(&req)
}
func getContent(url *url.URL, req *http.Request) *memcache.MemMapItem {
cacheToken := url.String()
cached := Repos.GetByKey(cacheToken)
if cached != nil {
return cached
}
backendUrl := getNewUrl(url)
newReq := http.Request{
Method: "GET",
RawURL: backendUrl.String(),
URL: backendUrl,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 0,
RemoteAddr: "192.168.0.21",
}
newReq.Header = http.Header{}
newReq.Header.Add("Accept", "*/*")
newReq.Header.Add("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.3")
newReq.Header.Add("Accept-Encoding", "utf-8")
newReq.Header.Add("Host", backendUrl.Host)
//newReq = ResponseWriter{};
response, err := Client.Do(&newReq)
if err != nil {
log.Fatal("error: ", err.String())
}
cacheItem := memcache.MemMapItem{Key: cacheToken}
cacheItem.Raw, _ = ioutil.ReadAll(response.Body)
cacheItem.Head = response.Header
Repos.Add(&cacheItem)
return &cacheItem
}
func send(req *http.Request) (resp *http.Response, err os.Error) {
if req.URL.Scheme != "http" {
return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
}
addr := req.URL.Host
if !hasPort(addr) {
addr += ":http"
}
info := req.URL.Userinfo
if len(info) > 0 {
enc := base64.URLEncoding
encoded := make([]byte, enc.EncodedLen(len(info)))
enc.Encode(encoded, strings.Bytes(info))
if req.Header == nil {
req.Header = make(map[string]string)
}
req.Header["Authorization"] = "Basic " + string(encoded)
}
conn, err := net.Dial("tcp", "", addr)
if err != nil {
return nil, err
}
err = req.Write(conn)
if err != nil {
conn.Close()
return nil, err
}
reader := bufio.NewReader(conn)
resp, err = http.ReadResponse(reader, req.Method)
if err != nil {
conn.Close()
return nil, err
}
resp.Body = readClose{resp.Body, conn}
return
}
func (conn *streamConn) connect() (*http.Response, os.Error) {
if conn.stale {
return nil, os.NewError("Stale connection")
}
var tcpConn net.Conn
var err os.Error
if proxy := os.Getenv("HTTP_PROXY"); len(proxy) > 0 {
proxy_url, _ := url.Parse(proxy)
tcpConn, err = net.Dial("tcp", proxy_url.Host)
} else {
tcpConn, err = net.Dial("tcp", conn.url.Host+":443")
}
if err != nil {
return nil, err
}
cf := &tls.Config{Rand: rand.Reader, Time: time.Nanoseconds}
ssl := tls.Client(tcpConn, cf)
conn.clientConn = http.NewClientConn(ssl, nil)
var req http.Request
req.URL = conn.url
req.Method = "GET"
req.Header = http.Header{}
req.Header.Set("Authorization", "Basic "+conn.authData)
if conn.postData != "" {
req.Method = "POST"
req.Body = nopCloser{bytes.NewBufferString(conn.postData)}
req.ContentLength = int64(len(conn.postData))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
resp, err := conn.clientConn.Do(&req)
if err != nil {
return nil, err
}
return resp, nil
}
// Post issues a POST to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPost(url, user, pwd, client, clientURL, version, agent, bodyType string,
body io.Reader) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "POST"
req.Body = body.(io.ReadCloser)
req.Header = map[string]string{
"Content-Type": bodyType,
"Transfer-Encoding": "chunked",
"User-Agent": agent,
"X-FluidDB-Client": client,
"X-FluidDB-Client-URL": clientURL,
"X-FluidDB-Version": version,
"Authorization": "Basic " + encodedUsernameAndPassword(user, pwd),
}
req.URL, err = http.ParseURL(url)
if err != nil {
return nil, err
}
return send(&req)
}
func main() {
url, err := http.ParseURL("http://bbs.golang-china.org/")
if err != nil {
log.Exit(err)
}
tcpConn, err := net.Dial("tcp", "", url.Host+":80")
if err != nil {
log.Exit(err)
}
clientConn := http.NewClientConn(tcpConn, nil)
var req http.Request
req.URL = url
req.Method = "GET"
req.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.11 (KHTML, like Gecko) Chrome/9.0.570.0 Safari/534.11"
req.Header = map[string]string{}
req.Header["Connection"] = "keep-alive"
err = clientConn.Write(&req)
if err != nil {
log.Exit(err)
}
resp, err := clientConn.Read()
if err != nil {
log.Exit(err)
}
defer resp.Body.Close()
log.Println("Http Response: " + resp.Status)
body, _ := ioutil.ReadAll(resp.Body)
log.Println(string(body))
}
// Post issues a POST to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPost(url, user, pwd, client, clientURL, version, agent, bodyType string,
body io.Reader) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "POST"
req.Body = body.(io.ReadCloser)
h := make(http.Header)
h.Add("Content-Type", bodyType)
h.Add("Transfer-Encoding", "chunked")
h.Add("User-Agent", agent)
h.Add("X-Twitter-Client", client)
h.Add("X-Twitter-Client-URL", clientURL)
h.Add("X-Twitter-Version", version)
h.Add("Authorization", "Basic "+encodedUsernameAndPassword(user, pwd))
req.Header = h
req.URL, err = http.ParseURL(url)
if err != nil {
return nil, err
}
return send(&req)
}
// PostForm issues a POST to the specified URL,
// with data's keys and values urlencoded as the request body.
//
// Caller should close r.Body when done reading from it.
func Post(t *Test) (r *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
var req http.Request
var url = t.Url
req.Method = "POST"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
var body *bytes.Buffer
var contentType string
if hasFile(&t.Param) || t.Method == "POST:mp" {
var boundary string
body, boundary = multipartBody(&t.Param)
contentType = "multipart/form-data; boundary=" + boundary
} else {
contentType = "application/x-www-form-urlencoded"
bodystr := http.EncodeQuery(t.Param)
body = bytes.NewBuffer([]byte(bodystr))
}
req.Body = nopCloser{body}
req.Header = http.Header{
"Content-Type": {contentType},
"Content-Length": {strconv.Itoa(body.Len())},
}
addHeadersAndCookies(&req, t)
req.ContentLength = int64(body.Len())
req.URL, err = http.ParseURL(url)
if err != nil {
return nil, url, cookies, err
}
debug("Will post to %s", req.URL.String())
r, finalUrl, cookies, err = DoAndFollow(&req, t.Dump)
return
}
// Downloads url and returns whatever result was.
// This function WILL NOT follow redirects.
func (w *Worker) Download(url *http.URL) (result *FetchResult) {
w.cl_lk.Lock()
client := w.clients[url.Host]
is_new_client := client == nil
if client == nil {
client = new(Client)
client.IOTimeout = w.IOTimeout
w.clients[url.Host] = client
}
w.cl_lk.Unlock()
req := new(http.Request)
req.URL = url
req.Header = make(map[string]string, 10)
req.UserAgent = "HeroshiBot/0.3 (+http://temoto.github.com/heroshi/; [email protected])"
result = client.FetchWithTimeout(req, w.FetchTimeout)
if is_new_client {
go w.staleClient(url.Host, w.KeepAlive)
}
return result
}
// Put issues a PUT to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPut(url, user, pwd, client, clientURL, version, agent, bodyType string,
body io.Reader) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "PUT"
req.Body = body.(io.ReadCloser)
if user != "" && pwd != "" {
req.Header = map[string][]string{
"Content-Type": {bodyType},
"Transfer-Encoding": {"chunked"},
"User-Agent": {agent},
"X-FluidDB-Client": {client},
"X-FluidDB-Client-URL": {clientURL},
"X-FluidDB-Version": {version},
"Authorization": {"Basic " + encodedUsernameAndPassword(user, pwd)},
}
} else {
req.Header = map[string][]string{
"Content-Type": {bodyType},
"Transfer-Encoding": {"chunked"},
"User-Agent": {agent},
"X-FluidDB-Client": {client},
"X-FluidDB-Client-URL": {clientURL},
"X-FluidDB-Version": {version},
}
}
req.URL, err = http.ParseURL(url)
if err != nil {
return nil, err
}
return send(&req)
}