Exemple #1
1
func TestRequestQueueLess(t *testing.T) {
	q := new(requestQueue)
	var a, b, c, d http.Request

	a.Header = map[string]string{
		"X-Pri": "1",
	}
	b.Header = map[string]string{
		"X-Pri": "2",
	}
	c.Header = map[string]string{
		"X-Pri": "1",
	}
	d.Header = map[string]string{} // default X-Pri: 5000

	q.Push(&clientRequest{r: &a})
	q.Push(&clientRequest{r: &b})
	q.Push(&clientRequest{r: &c})
	q.Push(&clientRequest{r: &d})

	if !q.Less(0, 1) {
		t.Error("want a < b")
	}
	if q.Less(1, 0) {
		t.Error("want b > a")
	}

	if !q.Less(2, 1) {
		t.Error("want c < b")
	}
	if q.Less(1, 2) {
		t.Error("want b > c")
	}

	if q.Less(0, 2) {
		t.Error("want a == c")
	}
	if q.Less(2, 0) {
		t.Error("want c == a")
	}

	if !q.Less(0, 3) {
		t.Error("want a < d")
	}
	if q.Less(3, 0) {
		t.Error("want d > a")
	}

}
Exemple #2
1
func (conn *streamConn) connect() (*http.Response, os.Error) {
	if conn.stale {
		return nil, os.NewError("Stale connection")
	}
	tcpConn, err := net.Dial("tcp", "", conn.url.Host+":80")
	if err != nil {
		return nil, err
	}
	conn.clientConn = http.NewClientConn(tcpConn, nil)

	var req http.Request
	req.URL = conn.url
	req.Method = "GET"
	req.Header = map[string]string{}
	req.Header["Authorization"] = "Basic " + conn.authData

	if conn.postData != "" {
		req.Method = "POST"
		req.Body = nopCloser{bytes.NewBufferString(conn.postData)}
		req.ContentLength = int64(len(conn.postData))
		req.Header["Content-Type"] = "application/x-www-form-urlencoded"
	}

	err = conn.clientConn.Write(&req)
	if err != nil {
		return nil, err
	}

	resp, err := conn.clientConn.Read()
	if err != nil {
		return nil, err
	}

	return resp, nil
}
Exemple #3
0
func post(url_ string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = "POST"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Close = true
	req.Header = map[string][]string{
		"Authorization": {"OAuth "},
	}
	req.TransferEncoding = []string{"chunked"}

	first := true
	for k, v := range oauthHeaders {
		if first {
			first = false
		} else {
			req.Header["Authorization"][0] += ",\n    "
		}
		req.Header["Authorization"][0] += k + "=\"" + v + "\""
	}

	req.URL, err = url.Parse(url_)
	if err != nil {
		return nil, err
	}

	return send(&req)
}
Exemple #4
0
// Much like http.Get. If s is nil, uses DefaultSender.
func Get(s Sender, url string) (rs []*http.Response, err os.Error) {
	for redirect := 0; ; redirect++ {
		if redirect >= 10 {
			err = os.ErrorString("stopped after 10 redirects")
			break
		}

		var req http.Request
		req.RawURL = url
		req.Header = map[string]string{}
		r, err := Send(s, &req)
		if err != nil {
			break
		}
		rs = prepend(r, rs)
		if shouldRedirect(r.StatusCode) {
			r.Body.Close()
			if url = r.GetHeader("Location"); url == "" {
				err = os.ErrorString(fmt.Sprintf("%d response missing Location header", r.StatusCode))
				break
			}
			continue
		}

		return
	}

	err = &http.URLError{"Get", url, err}
	return
}
Exemple #5
0
func NewRequest(method string, url string, doc IDocument) *http.Request {
	var req http.Request
	req.Method = method
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Close = true
	req.Header = map[string]string{
		"Content-Type":    "application/json",
		"X-Riak-ClientId": "riak.go",
	}
	if doc.VectorClock() != "" {
		req.Header["X-Riak-Vclock"] = doc.VectorClock()
	}
	req.TransferEncoding = []string{"chunked"}
	req.URL, _ = http.ParseURL(url)

	if doc.Json() != "" {
		cb := &ClosingBuffer{bytes.NewBufferString(doc.Json())}
		var rc io.ReadCloser
		rc = cb
		req.Body = rc
	}
	fmt.Println(req.URL)
	return &req
}
Exemple #6
0
func Delete(url string) *HttpRequestBuilder {
	var req http.Request
	req.Method = "DELETE"
	req.Header = http.Header{}
	req.Header.Set("User-Agent", defaultUserAgent)
	return &HttpRequestBuilder{url, &req, nil, map[string]string{}}
}
// sent a request off to twitter. Returns the response's body or an error.
func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) {
	req := new(http.Request)
	req.Method = method
	req.RawURL = url
	req.Host = URLHost
	req.Referer = "none"
	req.UserAgent = HTTPUserAgent
	req.Form = form
	req.Header = map[string]string{
		"Connection":    "Keep Alive",
		"Authorization": getAuthHeader(client),
	}
	req.Body = strings.NewReader(body)
	req.URL, err = http.ParseURL(req.RawURL)
	if err != nil {
		return "", err
	}

	// send request
	resp := new(http.Response)
	resp, err = http.Send(req)
	if err != nil {
		return "", err
	}
	result = getResponseBody(resp)
	return result, nil
}
Exemple #8
0
func post(theUrl string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
	var req http.Request
	var authorization string = "OAuth "
	req.Method = "POST"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Close = true
	req.Header = http.Header{}

	req.TransferEncoding = []string{"chunked"}

	first := true
	for k, v := range oauthHeaders {
		if first {
			first = false
		} else {
			authorization += ",\n    "
		}
		authorization += k + "=\"" + v + "\""
	}

	req.Header.Add("Authorization", authorization)

	req.URL, err = url.Parse(theUrl)
	if err != nil {
		return nil, err
	}

	return send(&req)
}
Exemple #9
0
// Perform a GET request for the test t.
func Get(t *Test) (r *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
	var url = t.Url // <-- Patched

	var req http.Request
	req.Method = "GET"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Header = http.Header{}

	if len(t.Param) > 0 {
		ep := http.EncodeQuery(t.Param)
		if strings.Contains(url, "?") {
			url = url + "&" + ep
		} else {
			url = url + "?" + ep
		}
	}
	req.URL, err = http.ParseURL(url)
	if err != nil {
		err = &http.URLError{"Get", url, err}
		return
	}

	addHeadersAndCookies(&req, t)
	url = req.URL.String()
	debug("Will get from %s", req.URL.String())
	r, finalUrl, cookies, err = DoAndFollow(&req, t.Dump)
	return
}
Exemple #10
0
func (c *Consumer) httpExecute(
	method string, url string, body string, oauthParams *OrderedParams) (*http.Response, os.Error) {

	if c.debug {
		fmt.Println("httpExecute(method: " + method + ", url: " + url)
	}

	var req http.Request
	req.Method = method
	req.Header = http.Header{}
	req.Body = newStringReadCloser(body)
	parsedurl, err := http.ParseURL(url)
	if err != nil {
		return nil, os.NewError("ParseUrl: " + err.String())
	}
	req.URL = parsedurl

	oauthHdr := "OAuth "
	for pos, key := range oauthParams.Keys() {
		if pos > 0 {
			oauthHdr += ","
		}
		oauthHdr += key + "=\"" + oauthParams.Get(key) + "\""
	}
	if c.debug {
		fmt.Println("AUTH-HDR: " + oauthHdr)
	}
	req.Header.Add("Authorization", oauthHdr)

	resp, err := c.HttpClient.Do(&req)

	if err != nil {
		return nil, os.NewError("Do: " + err.String())
	}

	debugHeader := ""
	for k, vals := range req.Header {
		for _, val := range vals {
			debugHeader += "[key: " + k + ", val: " + val + "]"
		}
	}

	if resp.StatusCode != http.StatusOK {
		bytes, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()

		return nil, os.NewError("HTTP response is not 200/OK as expected. Actual response: \n" +
			"\tResponse Status: '" + resp.Status + "'\n" +
			"\tResponse Code: " + strconv.Itoa(resp.StatusCode) + "\n" +
			"\tResponse Body: " + string(bytes) + "\n" +
			"\tRequst Headers: " + debugHeader)
	}
	return resp, err
}
Exemple #11
0
func sendBody(s Sender, url, method, bodyType string, body io.Reader) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = method
	req.RawURL = url
	req.Body = nopCloser{body}
	req.Header = map[string]string{
		"Content-Type": bodyType,
	}
	req.TransferEncoding = []string{"chunked"}
	return Send(s, &req)
}
Exemple #12
0
func Send(s Sender, req *http.Request) (resp *http.Response, err os.Error) {
	if s == nil {
		s = DefaultSender
	}
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	header := req.Header
	req.Header = map[string]string{}
	for k, v := range header {
		req.Header[http.CanonicalHeaderKey(k)] = v
	}
	return s.Send(req)
}
Exemple #13
0
func Get(url, user, pwd string) (r *http.Response, err os.Error) {
	var req http.Request

	req.Header = map[string]string{"Authorization": "Basic " +
		encodedUsernameAndPassword(user, pwd)}
	if req.URL, err = http.ParseURL(url); err != nil {
		return
	}
	if r, err = send(&req); err != nil {
		return
	}
	return
}
Exemple #14
0
func authGet(url, user, pwd string) (r *http.Response, err os.Error) {
	var req http.Request
	h := make(http.Header)
	h.Add("Authorization", "Basic "+
		encodedUsernameAndPassword(user, pwd))
	req.Header = h
	if req.URL, err = http.ParseURL(url); err != nil {
		return
	}
	if r, err = send(&req); err != nil {
		return
	}
	return
}
Exemple #15
0
// Head issues a HEAD to the specified URL.
func authHead(url, user, pwd string) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = "HEAD"
	if user != "" && pwd != "" {
		req.Header = map[string][]string{"Authorization": {"Basic " +
			encodedUsernameAndPassword(user, pwd)}}
	}
	if req.URL, err = http.ParseURL(url); err != nil {
		return
	}
	if r, err = send(&req); err != nil {
		return
	}
	return
}
// This function is launched async as a go routine. It tries to send a message
// to a remote server and sends a bool to nextChannel to indicate whether this
// has succeeded or not. It is not allowed to run this function multiple times in parallel
// for the same FederationGateway.
func (self *FederationGateway) sendFromQueue(req *WaveletUpdateRequest) {
	// No HTTP connection open yet?
	if self.connection == nil {
		con, err := net.Dial("tcp", "", fmt.Sprintf("%v:%v", self.manifest.HostName, self.manifest.Port))
		if err != nil {
			// TODO: Good error handling
			log.Println("Failed connecting to ", self.manifest, err)
			self.nextChannel <- false
			return
		}
		self.connection = http.NewClientConn(con, nil)
	}

	// Build the HTTP request
	var hreq http.Request
	hreq.Host = self.manifest.Domain
	hreq.Header = make(map[string]string)
	hreq.RawURL = fmt.Sprintf("http://%v:%v/fed%v", self.manifest.HostName, self.manifest.Port, req.uri)
	hreq.Method = "PUT"
	hreq.Body = NewRequestBody(req.Data)
	hreq.ContentLength = int64(len(req.Data))
	hreq.Header["Content-Type"] = req.MimeType
	log.Println("Sending WaveletUpdate to remote server ", hreq.RawURL)

	// Send the HTTP request
	self.connection.Write(&hreq)
	// Read the HTTP response
	hres, err := self.connection.Read()
	if err != nil {
		log.Println("Error reading HTTP response from ", self.manifest, err)
		// TODO: Better error handling
		self.connection.Close()
		self.connection = nil
		self.nextChannel <- false
		return
	}

	log.Println("After sending WaveletUpdate, status code is ", hres.StatusCode)
	// Success in sending the wavelet update?
	if hres.StatusCode == 200 {
		self.nextChannel <- true
		return
	}
	// Sending the wavelet update failed
	self.nextChannel <- false
}
Exemple #17
0
func YadisRequest(url_ string, method string) (resp *http.Response, err os.Error) {
	resp = nil

	var request = new(http.Request)
	var client = new(http.Client)
	var Header = make(http.Header)

	request.Method = method
	request.RawURL = url_

	request.URL, err = url.Parse(url_)
	if err != nil {
		return
	}

	// Common parameters
	request.Proto = "HTTP/1.0"
	request.ProtoMajor = 1
	request.ProtoMinor = 0
	request.ContentLength = 0
	request.Close = true

	Header.Add("Accept", "application/xrds+xml")
	request.Header = Header

	// Follow a maximum of 5 redirections
	for i := 0; i < 5; i++ {
		response, err := client.Do(request)

		if err != nil {
			return nil, err
		}
		if response.StatusCode == 301 || response.StatusCode == 302 || response.StatusCode == 303 || response.StatusCode == 307 {
			location := response.Header.Get("Location")
			request.RawURL = location
			request.URL, err = url.Parse(location)
			if err != nil {
				return
			}
		} else {
			return response, nil
		}
	}
	return nil, os.NewError("Too many redirections")
}
Exemple #18
0
func nmcPOST(body *bytes.Buffer) (response *http.Response, err os.Error) {
	client := http.Client{}

	var req http.Request
	req.Method = "POST"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Close = true
	req.Body = ioutil.NopCloser(body)
	req.Header = http.Header{
		"Content-Type":   []string{"text/plain"},
		"Content-Length": []string{strconv.Itoa(body.Len())},
	}
	req.ContentLength = int64(body.Len())
	req.URL = options.nmcURL

	return client.Do(&req)
}
Exemple #19
0
func (client *Client) Fetch(req *http.Request) (result *FetchResult) {
	creds := req.URL.RawUserinfo
	_, has_auth_header := req.Header["Authorization"]
	if len(creds) > 0 && !has_auth_header {
		encoded := make([]byte, base64.URLEncoding.EncodedLen(len(creds)))
		base64.URLEncoding.Encode(encoded, []byte(creds))
		if req.Header == nil {
			req.Header = make(map[string]string)
		}
		req.Header["Authorization"] = "Basic " + string(encoded)
	}

	// debug
	if false {
		dump, _ := http.DumpRequest(req, true)
		print(string(dump))
	}

	resp, err := client.Request(req)
	if err != nil {
		return ErrorResult(req.URL.Raw, err.String())
	}

	// Prevent simultaneous IO from different goroutines.
	client.lk.Lock()
	defer client.lk.Unlock()

	var buf bytes.Buffer
	_, err = io.Copy(&buf, resp.Body)
	responseBody := buf.Bytes()
	if err != nil {
		return ErrorResult(req.URL.Raw, err.String())
	}
	resp.Body.Close()

	return &FetchResult{
		Url:        req.URL.Raw,
		Success:    true,
		Status:     resp.Status,
		StatusCode: resp.StatusCode,
		Body:       responseBody,
		Headers:    resp.Header,
	}
}
Exemple #20
0
func (ds *dummySender) Send(req *http.Request) (*http.Response, os.Error) {
	if req.Header == nil {
		req.Header = map[string]string{}
	}
	if etag, ok := req.Header["If-None-Match"]; ok {
		if resp, ok := testEtags[etag]; ok {
			resp.Status = "304 Not Modified"
			resp.StatusCode = 304
			return resp, nil
		}
	}
	resp, ok := dummyResponses[req.RawURL]
	if !ok {
		return nil, os.NewError("no response")
	}
	resp.Status = "200 OK"
	resp.StatusCode = 200
	return resp, nil
}
Exemple #21
0
// Post issues a POST to the specified URL.
//
// Caller should close r.Body when done reading it.
func Post(url, user, pwd, bodyType string, body io.Reader) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = "POST"
	req.Body = nopCloser{body}
	req.Header = map[string]string{
		"Content-Type":      bodyType,
		"Transfer-Encoding": "chunked",
		"X-Twitter-Client":  "gotweet",
		"X-Twitter-Version": "0.1",
		"Authorization":     "Basic " + encodedUsernameAndPassword(user, pwd),
	}

	req.URL, err = http.ParseURL(url)
	if err != nil {
		return nil, err
	}

	return send(&req)
}
Exemple #22
0
func getContent(url *url.URL, req *http.Request) *memcache.MemMapItem {

	cacheToken := url.String()

	cached := Repos.GetByKey(cacheToken)
	if cached != nil {
		return cached
	}
	backendUrl := getNewUrl(url)

	newReq := http.Request{
		Method:     "GET",
		RawURL:     backendUrl.String(),
		URL:        backendUrl,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 0,
		RemoteAddr: "192.168.0.21",
	}

	newReq.Header = http.Header{}
	newReq.Header.Add("Accept", "*/*")
	newReq.Header.Add("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.3")
	newReq.Header.Add("Accept-Encoding", "utf-8")
	newReq.Header.Add("Host", backendUrl.Host)

	//newReq = ResponseWriter{};

	response, err := Client.Do(&newReq)

	if err != nil {
		log.Fatal("error: ", err.String())
	}

	cacheItem := memcache.MemMapItem{Key: cacheToken}
	cacheItem.Raw, _ = ioutil.ReadAll(response.Body)
	cacheItem.Head = response.Header

	Repos.Add(&cacheItem)

	return &cacheItem
}
Exemple #23
0
func send(req *http.Request) (resp *http.Response, err os.Error) {
	if req.URL.Scheme != "http" {
		return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
	}

	addr := req.URL.Host
	if !hasPort(addr) {
		addr += ":http"
	}
	info := req.URL.Userinfo
	if len(info) > 0 {
		enc := base64.URLEncoding
		encoded := make([]byte, enc.EncodedLen(len(info)))
		enc.Encode(encoded, strings.Bytes(info))
		if req.Header == nil {
			req.Header = make(map[string]string)
		}
		req.Header["Authorization"] = "Basic " + string(encoded)
	}
	conn, err := net.Dial("tcp", "", addr)
	if err != nil {
		return nil, err
	}

	err = req.Write(conn)
	if err != nil {
		conn.Close()
		return nil, err
	}

	reader := bufio.NewReader(conn)
	resp, err = http.ReadResponse(reader, req.Method)
	if err != nil {
		conn.Close()
		return nil, err
	}

	resp.Body = readClose{resp.Body, conn}

	return
}
Exemple #24
0
func (conn *streamConn) connect() (*http.Response, os.Error) {
	if conn.stale {
		return nil, os.NewError("Stale connection")
	}
	var tcpConn net.Conn
	var err os.Error
	if proxy := os.Getenv("HTTP_PROXY"); len(proxy) > 0 {
		proxy_url, _ := url.Parse(proxy)
		tcpConn, err = net.Dial("tcp", proxy_url.Host)
	} else {
		tcpConn, err = net.Dial("tcp", conn.url.Host+":443")
	}
	if err != nil {
		return nil, err
	}
	cf := &tls.Config{Rand: rand.Reader, Time: time.Nanoseconds}
	ssl := tls.Client(tcpConn, cf)

	conn.clientConn = http.NewClientConn(ssl, nil)

	var req http.Request
	req.URL = conn.url
	req.Method = "GET"
	req.Header = http.Header{}
	req.Header.Set("Authorization", "Basic "+conn.authData)

	if conn.postData != "" {
		req.Method = "POST"
		req.Body = nopCloser{bytes.NewBufferString(conn.postData)}
		req.ContentLength = int64(len(conn.postData))
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	resp, err := conn.clientConn.Do(&req)
	if err != nil {
		return nil, err
	}

	return resp, nil
}
Exemple #25
0
// Post issues a POST to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPost(url, user, pwd, client, clientURL, version, agent, bodyType string,
	body io.Reader) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = "POST"
	req.Body = body.(io.ReadCloser)
	req.Header = map[string]string{
		"Content-Type":         bodyType,
		"Transfer-Encoding":    "chunked",
		"User-Agent":           agent,
		"X-FluidDB-Client":     client,
		"X-FluidDB-Client-URL": clientURL,
		"X-FluidDB-Version":    version,
		"Authorization":        "Basic " + encodedUsernameAndPassword(user, pwd),
	}

	req.URL, err = http.ParseURL(url)
	if err != nil {
		return nil, err
	}

	return send(&req)
}
Exemple #26
0
func main() {

	url, err := http.ParseURL("http://bbs.golang-china.org/")

	if err != nil {
		log.Exit(err)
	}

	tcpConn, err := net.Dial("tcp", "", url.Host+":80")

	if err != nil {
		log.Exit(err)
	}

	clientConn := http.NewClientConn(tcpConn, nil)

	var req http.Request
	req.URL = url
	req.Method = "GET"
	req.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.11 (KHTML, like Gecko) Chrome/9.0.570.0 Safari/534.11"
	req.Header = map[string]string{}
	req.Header["Connection"] = "keep-alive"

	err = clientConn.Write(&req)
	if err != nil {
		log.Exit(err)
	}
	resp, err := clientConn.Read()
	if err != nil {
		log.Exit(err)
	}
	defer resp.Body.Close()

	log.Println("Http Response: " + resp.Status)
	body, _ := ioutil.ReadAll(resp.Body)

	log.Println(string(body))
}
Exemple #27
0
// Post issues a POST to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPost(url, user, pwd, client, clientURL, version, agent, bodyType string,
	body io.Reader) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = "POST"
	req.Body = body.(io.ReadCloser)

	h := make(http.Header)
	h.Add("Content-Type", bodyType)
	h.Add("Transfer-Encoding", "chunked")
	h.Add("User-Agent", agent)
	h.Add("X-Twitter-Client", client)
	h.Add("X-Twitter-Client-URL", clientURL)
	h.Add("X-Twitter-Version", version)
	h.Add("Authorization", "Basic "+encodedUsernameAndPassword(user, pwd))
	req.Header = h

	req.URL, err = http.ParseURL(url)
	if err != nil {
		return nil, err
	}

	return send(&req)
}
Exemple #28
0
// PostForm issues a POST to the specified URL,
// with data's keys and values urlencoded as the request body.
//
// Caller should close r.Body when done reading from it.
func Post(t *Test) (r *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
	var req http.Request
	var url = t.Url
	req.Method = "POST"
	req.ProtoMajor = 1
	req.ProtoMinor = 1
	req.Close = true
	var body *bytes.Buffer
	var contentType string
	if hasFile(&t.Param) || t.Method == "POST:mp" {
		var boundary string
		body, boundary = multipartBody(&t.Param)
		contentType = "multipart/form-data; boundary=" + boundary
	} else {
		contentType = "application/x-www-form-urlencoded"
		bodystr := http.EncodeQuery(t.Param)
		body = bytes.NewBuffer([]byte(bodystr))
	}

	req.Body = nopCloser{body}
	req.Header = http.Header{
		"Content-Type":   {contentType},
		"Content-Length": {strconv.Itoa(body.Len())},
	}
	addHeadersAndCookies(&req, t)

	req.ContentLength = int64(body.Len())
	req.URL, err = http.ParseURL(url)
	if err != nil {
		return nil, url, cookies, err
	}
	debug("Will post to %s", req.URL.String())

	r, finalUrl, cookies, err = DoAndFollow(&req, t.Dump)
	return
}
Exemple #29
0
// Downloads url and returns whatever result was.
// This function WILL NOT follow redirects.
func (w *Worker) Download(url *http.URL) (result *FetchResult) {
	w.cl_lk.Lock()
	client := w.clients[url.Host]
	is_new_client := client == nil
	if client == nil {
		client = new(Client)
		client.IOTimeout = w.IOTimeout
		w.clients[url.Host] = client
	}
	w.cl_lk.Unlock()

	req := new(http.Request)
	req.URL = url
	req.Header = make(map[string]string, 10)
	req.UserAgent = "HeroshiBot/0.3 (+http://temoto.github.com/heroshi/; [email protected])"

	result = client.FetchWithTimeout(req, w.FetchTimeout)

	if is_new_client {
		go w.staleClient(url.Host, w.KeepAlive)
	}

	return result
}
Exemple #30
-1
// Put issues a PUT to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPut(url, user, pwd, client, clientURL, version, agent, bodyType string,
	body io.Reader) (r *http.Response, err os.Error) {
	var req http.Request
	req.Method = "PUT"
	req.Body = body.(io.ReadCloser)
	if user != "" && pwd != "" {
		req.Header = map[string][]string{
			"Content-Type":         {bodyType},
			"Transfer-Encoding":    {"chunked"},
			"User-Agent":           {agent},
			"X-FluidDB-Client":     {client},
			"X-FluidDB-Client-URL": {clientURL},
			"X-FluidDB-Version":    {version},
			"Authorization":        {"Basic " + encodedUsernameAndPassword(user, pwd)},
		}
	} else {
		req.Header = map[string][]string{
			"Content-Type":         {bodyType},
			"Transfer-Encoding":    {"chunked"},
			"User-Agent":           {agent},
			"X-FluidDB-Client":     {client},
			"X-FluidDB-Client-URL": {clientURL},
			"X-FluidDB-Version":    {version},
		}
	}

	req.URL, err = http.ParseURL(url)
	if err != nil {
		return nil, err
	}

	return send(&req)
}