func GetQueue() (Q *sqs.Queue, err os.Error) {
proxyURL, err := GetProxyURL()
if err != nil {
return
}
id, err := GetAWSIdentity()
if err != nil {
return
}
if queueURL != nil && *queueURL != "" {
qrl, err := http.ParseURL(*queueURL)
if err == nil {
ep := awsconn.NewEndpoint(qrl, proxyURL)
Q = sqs.NewQueueURL(ep)
}
} else if queueName != nil && *queueName != "" {
ep, err := http.ParseURL(*sqsEndpoint)
if err == nil {
log.Printf("Parsed EP url: %v", ep)
ep := awsconn.NewEndpoint(ep, proxyURL)
_sqs := sqs.NewService(ep)
Q, err = _sqs.CreateQueue(id, *queueName, 90)
if err != nil {
log.Fatalf("Qerr: [%v]", err)
}
}
} else {
err = os.NewError("Either Queue(+Endpoint) or QueueURL are required")
}
return
}
func NewWaveUrl(url string) (result *WaveUrl, err os.Error) {
u, err := http.ParseURL(url)
if err != nil {
return nil, err
}
if u.Scheme != "wave" {
return nil, os.NewError("Not a wave url")
}
result = &WaveUrl{WaveletDomain: u.Host}
wave := u.Path[1:]
i := strings.Index(wave, "/")
if i == -1 || i == len(wave)-1 {
return nil, os.NewError("Malformed wave url")
}
result.WaveletId = wave[i+1:]
wave = wave[:i]
i = strings.Index(wave, "$")
if i == -1 {
result.WaveDomain = result.WaveletDomain
result.WaveId = wave
} else {
result.WaveDomain = wave[:i]
result.WaveId = wave[i+1:]
}
// TODO: Check that only allowed characters are used
return result, nil
}
func (self *Federation) Forward(blobref string, users []string) {
// Determine the servers that have to be informed
urls := make(map[string]vec.StringVector)
for _, user := range users {
if user == self.userID {
continue
}
rawurl, err := self.ns.Lookup(user)
if err != nil {
log.Print("Unknown user", err)
continue
}
_, err = http.ParseURL(rawurl)
if err != nil {
log.Printf("Malformed URL: %v\n", rawurl)
continue
}
urlList, _ := urls[rawurl]
urlList.Push(user[strings.Index(user, "@")+1:])
urls[rawurl] = urlList
}
if len(urls) > 0 {
log.Printf("Forwarding %v to %v\n", blobref, users)
}
for url, urlUsers := range urls {
q := self.getQueue(url)
q <- queueEntry{urlUsers, blobref}
}
}
func TestGetConnectionMethodWorks(*testing.T) {
drv := new(DummyDriver)
func(d Driver) {
url, _ := http.ParseURL("foo://root@localhost/testdb")
_, _ = d.Connect(url)
}(drv)
}
func NewRequest(method string, url string, doc IDocument) *http.Request {
var req http.Request
req.Method = method
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Close = true
req.Header = map[string]string{
"Content-Type": "application/json",
"X-Riak-ClientId": "riak.go",
}
if doc.VectorClock() != "" {
req.Header["X-Riak-Vclock"] = doc.VectorClock()
}
req.TransferEncoding = []string{"chunked"}
req.URL, _ = http.ParseURL(url)
if doc.Json() != "" {
cb := &ClosingBuffer{bytes.NewBufferString(doc.Json())}
var rc io.ReadCloser
rc = cb
req.Body = rc
}
fmt.Println(req.URL)
return &req
}
/*
Dial opens a new client connection to a Web Socket.
A trivial example client:
package main
import (
"websocket"
"strings"
)
func main() {
ws, err := websocket.Dial("ws://localhost/ws", "", "http://localhost/");
if err != nil {
panic("Dial: " + err.String())
}
if _, err := ws.Write([]byte("hello, world!\n")); err != nil {
panic("Write: " + err.String())
}
var msg = make([]byte, 512);
if n, err := ws.Read(msg); err != nil {
panic("Read: " + err.String())
}
// use msg[0:n]
}
*/
func Dial(url, protocol, origin string) (ws *Conn, err os.Error) {
var client net.Conn
parsedUrl, err := http.ParseURL(url)
if err != nil {
goto Error
}
switch parsedUrl.Scheme {
case "ws":
client, err = net.Dial("tcp", "", parsedUrl.Host)
case "wss":
client, err = tls.Dial("tcp", "", parsedUrl.Host)
default:
err = ErrBadScheme
}
if err != nil {
goto Error
}
ws, err = newClient(parsedUrl.RawPath, parsedUrl.Host, origin, url, protocol, client, handshake)
if err != nil {
goto Error
}
return
Error:
return nil, &DialError{url, protocol, origin, err}
}
func (w *Worker) AskRobots(url *http.URL) (bool, *FetchResult) {
robots_url_str := fmt.Sprintf("%s://%s/robots.txt", url.Scheme, url.Host)
robots_url, err := http.ParseURL(robots_url_str)
if err != nil {
return false, ErrorResult(url.Raw, err.String())
}
fetch_result := w.Fetch(robots_url)
if !fetch_result.Success {
fetch_result.Status = "Robots download error: " + fetch_result.Status
return false, fetch_result
}
var robots *robotstxt.RobotsData
robots, err = robotstxt.FromResponseBytes(fetch_result.StatusCode, fetch_result.Body, false)
if err != nil {
fetch_result.Status = "Robots parse error: " + err.String()
return false, fetch_result
}
robots.DefaultAgent = "HeroshiBot"
var allow bool
allow, err = robots.Test(url.RawPath)
if err != nil {
return false, ErrorResult(url.Raw, "Robots test error: "+err.String())
}
if !allow {
return allow, ErrorResult(url.Raw, "Robots disallow")
}
return allow, nil
}
// List all queues available at an endpoint.
func (self *Service) ListQueues(id cryptools.NamedSigner, prefix string) (out []*Queue, err os.Error) {
sqsReq, err := self.signedRequest(id, "/", map[string]string{
"Action": "ListQueues",
})
if err != nil {
return
}
xresp := &listQueuesResponse{}
xerr := &errorResponse{}
err = self.Endpoint.SendParsable(sqsReq, xresp, xerr)
if err != nil {
return
}
out = make([]*Queue, len(xresp.ListQueuesResult.QueueUrl))
for i := range xresp.ListQueuesResult.QueueUrl {
url, err := http.ParseURL(xresp.ListQueuesResult.QueueUrl[i])
if err != nil {
break
}
ep := awsconn.NewEndpoint(url, self.Endpoint.ProxyURL)
out[i] = NewQueueURL(ep)
}
//log.Printf("ListQueue: %v", out)
return
}
// sent a request off to twitter. Returns the response's body or an error.
func send(url, method string, form map[string][]string, client *Client, body string) (result string, err os.Error) {
req := new(http.Request)
req.Method = method
req.RawURL = url
req.Host = URLHost
req.Referer = "none"
req.UserAgent = HTTPUserAgent
req.Form = form
req.Header = map[string]string{
"Connection": "Keep Alive",
"Authorization": getAuthHeader(client),
}
req.Body = strings.NewReader(body)
req.URL, err = http.ParseURL(req.RawURL)
if err != nil {
return "", err
}
// send request
resp := new(http.Response)
resp, err = http.Send(req)
if err != nil {
return "", err
}
result = getResponseBody(resp)
return result, nil
}
func TestURL(t *testing.T) {
for _, ex := range examples {
req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(ex.request)))
if err != nil {
t.Error(err)
}
if len(req.Header) > 1 {
// Only works on requests without any headers
// Host header has been removed
continue
}
date := req.Header.Get("Date")
url, err := http.ParseURL("http://" + req.Host + req.URL.RawPath)
if err != nil {
t.Error(err)
}
signed, err := URL(url, AWSAccessKeyId, AWSSecretAccessKey, req.Method, date)
if err != nil {
t.Error(err)
}
// Fudge a bit on this test. The date should be in Unix timestamp
// but we can only compare against known signatures for now so use
// the existing date.
if auth := addAuth(url.Raw, ex.auth, date); auth != signed.Raw {
t.Error("Fail URL:", ex.name, "got:", signed.Raw, "want:", auth, "headers:", len(req.Header))
}
}
}
func buildTestRequest(method string, path string, body string, headers map[string]string) *Request {
host := "127.0.0.1"
port := "80"
rawurl := "http://" + host + ":" + port + path
url, _ := http.ParseURL(rawurl)
proto := "HTTP/1.1"
useragent := "web.go test framework"
if headers == nil {
headers = map[string]string{}
}
if method == "POST" {
headers["Content-Length"] = fmt.Sprintf("%d", len(body))
headers["Content-Type"] = "text/plain"
}
req := Request{Method: method,
RawURL: rawurl,
URL: url,
Proto: proto,
Host: host,
UserAgent: useragent,
Headers: headers,
Body: bytes.NewBufferString(body),
}
return &req
}
// Create a queue, returning the Queue object.
func (self *Service) CreateQueue(id *aws.Signer, name string, dvtimeout int) (mq *Queue, err os.Error) {
var resp *http.Response
parms := http.Values{}
parms.Set("Action", "CreateQueue")
parms.Set("QueueName", name)
parms.Set("DefaultVisibilityTimeout", strconv.Itoa(dvtimeout))
req := aws.NewRequest(self.URL, "GET", nil, parms)
err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15)
if err == nil {
resp, err = self.conn.Request(req)
if err == nil {
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
xmlresp := createQueueResponse{}
err = xml.Unmarshal(resp.Body, &xmlresp)
if err == nil {
var qrl *http.URL
qrl, err = http.ParseURL(xmlresp.QueueURL)
if err == nil {
mq = NewQueue(qrl)
}
}
} else {
err = os.NewError("Unexpected response")
}
}
}
return
}
// Perform a GET request for the test t.
func Get(t *Test) (r *http.Response, finalUrl string, cookies []*http.Cookie, err os.Error) {
var url = t.Url // <-- Patched
var req http.Request
req.Method = "GET"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Header = http.Header{}
if len(t.Param) > 0 {
ep := http.EncodeQuery(t.Param)
if strings.Contains(url, "?") {
url = url + "&" + ep
} else {
url = url + "?" + ep
}
}
req.URL, err = http.ParseURL(url)
if err != nil {
err = &http.URLError{"Get", url, err}
return
}
addHeadersAndCookies(&req, t)
url = req.URL.String()
debug("Will get from %s", req.URL.String())
r, finalUrl, cookies, err = DoAndFollow(&req, t.Dump)
return
}
func ParseURL(url string) (parsed_url *http.URL) {
parsed_url, err := http.ParseURL(url)
if err != nil {
log.Fatal(err)
}
return parsed_url
}
func dial(host string) (conn *http.ClientConn) {
var tcp net.Conn
var err os.Error
fmt.Fprintf(os.Stderr, "http-gonsole: establishing a TCP connection ...\n")
proxy := os.Getenv("HTTP_PROXY")
if len(proxy) > 0 {
proxy_url, _ := http.ParseURL(proxy)
tcp, err = net.Dial("tcp", proxy_url.Host)
} else {
tcp, err = net.Dial("tcp", host)
}
if err != nil {
fmt.Fprintln(os.Stderr, "http-gonsole:", err)
os.Exit(1)
}
if *useSSL {
cf := &tls.Config{Rand: rand.Reader, Time: time.Nanoseconds}
ssl := tls.Client(tcp, cf)
conn = http.NewClientConn(ssl, nil)
if len(proxy) > 0 {
tcp.Write([]byte("CONNECT " + host + " HTTP/1.0\r\n\r\n"))
b := make([]byte, 1024)
tcp.Read(b)
}
} else {
conn = http.NewClientConn(tcp, nil)
}
return
}
func main() {
scheme := "http"
host := "localhost:80"
headers := make(http.Header)
cookies := new([]*Cookie)
p := "/"
flag.Parse()
if flag.NArg() > 0 {
tmp := flag.Arg(0)
if match, _ := regexp.MatchString("^[^:]+(:[0-9]+)?$", tmp); match {
tmp = "http://" + tmp
}
targetURL, err := http.ParseURL(tmp)
if err != nil {
fmt.Fprintln(os.Stderr, "malformed URL")
os.Exit(-1)
}
host = targetURL.Host
if len(host) == 0 {
fmt.Fprintln(os.Stderr, "invalid host name")
os.Exit(-1)
}
if match, _ := regexp.MatchString("^[^:]+:[0-9]+$", host); !match {
host = host + ":80"
}
if *useSSL || targetURL.Scheme == "https" {
*useSSL = true
scheme = "https"
}
scheme = targetURL.Scheme
info := targetURL.RawUserinfo
if len(info) > 0 {
enc := base64.URLEncoding
encoded := make([]byte, enc.EncodedLen(len(info)))
enc.Encode(encoded, []byte(info))
headers.Set("Authorization", "Basic "+string(encoded))
}
p = strings.Replace(path.Clean(targetURL.Path), "\\", "/", -1)
if p == "." {
p = "/"
}
} else if *useSSL {
scheme = "https"
host = "localhost:443"
}
headers.Set("Host", host)
session := &Session{
scheme: scheme,
host: host,
conn: dial(host),
headers: headers,
cookies: cookies,
path: &p,
}
defer session.conn.Close()
done := false
for !done {
done = session.repl()
}
}
func GetEndpoint(region string, proxy *http.URL) (ep *awsconn.Endpoint, err os.Error) {
url, err := http.ParseURL("http://elasticloadbalancing." + region + ".amazonaws.com")
if err == nil {
ep = awsconn.NewEndpoint(url, proxy)
}
return
}
func (pf *PacmanFetcher) Fetch(pkgname string) ([]string, FetchError) {
urltext, err := pf.findPackageUrl(pkgname)
if err != nil {
return nil, err
}
url, oserr := http.ParseURL(urltext)
if oserr != nil {
return nil, FetchErrorWrap(pkgname, oserr)
}
var pkgpath string
switch url.Scheme {
case "http":
fallthrough
case "https":
pkgpath, oserr = pf.httpDownload(url)
case "ftp":
pkgpath, oserr = pf.ftpDownload(url)
default:
return nil, NewFetchError(pkgname, "Unrecognized URL scheme: "+url.Scheme)
}
if oserr != nil {
return nil, FetchErrorWrap(pkgname, oserr)
}
return []string{pkgpath}, nil
}
// Create a queue, returning the Queue object.
func (self *Service) CreateQueue(id cryptools.NamedSigner, name string, dvtimeout int) (mq *Queue, err os.Error) {
sqsReq, err := self.signedRequest(id, "/", map[string]string{
"Action": "CreateQueue",
"QueueName": name,
"DefaultVisibilityTimeout": strconv.Itoa(dvtimeout),
})
if err != nil {
return
}
xresp := &createQueueResponse{}
xerr := &errorResponse{}
err = self.Endpoint.SendParsable(sqsReq, xresp, xerr)
if err != nil {
return
}
qrl, err := http.ParseURL(xresp.CreateQueueResult.QueueUrl)
if err != nil {
return
}
ep := awsconn.NewEndpoint(qrl, self.Endpoint.ProxyURL)
mq = NewQueueURL(ep)
return
}
func HttpGet(url string) (*http.Response, os.Error) {
var r *http.Response
var err os.Error
if proxy := os.Getenv("HTTP_PROXY"); len(proxy) > 0 {
proxy_url, _ := http.ParseURL(proxy)
tcp, _ := net.Dial("tcp", "", proxy_url.Host)
conn := http.NewClientConn(tcp, nil)
var req http.Request
req.URL, _ = http.ParseURL(url)
req.Method = "GET"
err = conn.Write(&req)
r, err = conn.Read()
} else {
r, _, err = http.Get(url)
}
return r, err
}
func TestBuildAuthUrlNoAuth(t *testing.T) {
url, err := goget.BuildAuthUrl("http://example.com", "", "")
AssertNil(t, err)
urlObj, err := http.ParseURL(url)
AssertNil(t, err)
AssertEquals(t, "", urlObj.Userinfo)
}
func NewTHttpPostClient(url string) (TTransport, os.Error) {
parsedURL, err := http.ParseURL(url)
if err != nil {
return nil, err
}
buf := make([]byte, 0, 1024)
return &THttpClient{url: parsedURL, requestBuffer: bytes.NewBuffer(buf)}, nil
}
func (c *conn) prepare() (err os.Error) {
method, rawURL, version, err := parseRequestLine(c.br)
if err != nil {
return err
}
header, err := parseHeader(c.br)
if err != nil {
return err
}
url, err := http.ParseURL(rawURL)
if err != nil {
return err
}
if url.Host == "" {
url.Host = header.GetDef(web.HeaderHost, "")
if url.Host == "" {
url.Host = c.serverName
}
}
if c.secure {
url.Scheme = "https"
} else {
url.Scheme = "http"
}
req, err := web.NewRequest(c.netConn.RemoteAddr().String(), method, url, version, header)
if err != nil {
return
}
c.req = req
c.requestAvail = req.ContentLength
if c.requestAvail < 0 {
c.requestAvail = 0
}
if s, found := req.Header.Get(web.HeaderExpect); found {
c.write100Continue = strings.ToLower(s) == "100-continue"
}
connection := strings.ToLower(req.Header.GetDef(web.HeaderConnection, ""))
if version >= web.ProtocolVersion(1, 1) {
c.closeAfterResponse = connection == "close"
} else if version == web.ProtocolVersion(1, 0) && req.ContentLength >= 0 {
c.closeAfterResponse = connection != "keep-alive"
} else {
c.closeAfterResponse = true
}
req.Responder = c
req.Body = requestReader{c}
return nil
}
func (bucket *Bucket) GetFile(file *os.FileInfo) (err os.Error) {
url, e := http.ParseURL(bucket.Url + "/" + file.Name)
if e != nil {
return err
}
req := &http.Request{Method: "GET", RawURL: bucket.Url + "/" + file.Name, URL: url, Header: make(map[string]string)}
date := time.LocalTime().Format(time.RFC1123)
req.Header["Date"] = date
bucket.sign(req)
//req.Write(os.Stdout)
conn, err := net.Dial("tcp", "", req.URL.Host+":"+req.URL.Scheme)
if err != nil {
return err
}
req.Write(conn)
reader := bufio.NewReader(conn)
resp, _ := http.ReadResponse(reader, req.Method)
//resp.Write(os.Stdout)
filename := strings.TrimLeft(url.Path, "/")
path := strings.SplitAfter(filename, "/", -1)
//pid := os.GetPid()
var b []byte
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
mode := (file.Mode & 0777)
fmt.Printf("0%o\n", mode)
//var dirmode uint32
//dirmode = (syscall.S_IFDIR^0666)&0777
resp.Body.Close()
//create file or dir
for i := 0; i < len(path); i++ {
f := i + 1
if f == len(path) {
openfile, err := os.Open(filename, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_EXCL, mode)
if err != nil {
return err
} else {
openfile.Chown(file.Uid, file.Gid)
openfile.Write(b)
openfile.Close()
os.Chtimes(filename, file.Mtime_ns, file.Mtime_ns)
}
} else {
err := os.Mkdir(path[i], 0755)
if err != nil {
return err
}
}
}
return nil
}
// Parse a response-Location-URI to get the ID of the worked-on snip
func (client *Client) IdFromURL(urlString string) (string, os.Error) {
var url *http.URL
var err os.Error
if url, err = http.ParseURL(urlString); err != nil {
return "", err
}
return string(url.Path[len(client.resource.Path):]), nil
}
func oauthGet(accessToken string, urlString string) (*http.Response, os.Error) {
url, _ := http.ParseURL(urlString)
conn, _ := net.Dial("tcp", url.Host+":80")
clientConn := http.NewClientConn(conn, nil)
header := map[string][]string{"Authorization": {"OAuth " + accessToken}}
request := http.Request{Method: "GET", URL: url, Header: header}
clientConn.Write(&request)
return clientConn.Read(&request)
}
func NewRequest(url, authToken, method string) (req *http.Request) {
req = new(http.Request)
req.RawURL = url
req.URL, _ = http.ParseURL(req.RawURL)
req.Method = method
req.Header = make(map[string][]string)
// Set the authorization header with the token (required for picasa authentication).
req.Header.Add("Authorization", "AuthSub token="+authToken+"")
return
}
// Convenience method to clean up calls.
func DefaultS3Service() (id *aws.Signer, s *s3.Service, err os.Error) {
id, err = DefaultSigner()
if err == nil {
url, err := http.ParseURL(flag_endpoint_url)
if err == nil {
s = s3.NewService(url)
}
}
return
}
// UploadURL creates an upload URL for the form that the user will fill out,
// passing the application path to load when the POST of the form is
// completed. These URLs expire and should not be reused.
func UploadURL(c appengine.Context, successPath string) (*http.URL, os.Error) {
req := &pb.CreateUploadURLRequest{
SuccessPath: proto.String(successPath),
}
res := &pb.CreateUploadURLResponse{}
if err := c.Call("blobstore", "CreateUploadURL", req, res); err != nil {
return nil, err
}
return http.ParseURL(*res.Url)
}
// Put issues a PUT to the specified URL.
//
// Caller should close r.Body when done reading it.
func authPut(url, user, pwd, client, clientURL, version, agent, bodyType string,
body io.Reader) (r *http.Response, err os.Error) {
var req http.Request
req.Method = "PUT"
req.Body = body.(io.ReadCloser)
if user != "" && pwd != "" {
req.Header = map[string][]string{
"Content-Type": {bodyType},
"Transfer-Encoding": {"chunked"},
"User-Agent": {agent},
"X-FluidDB-Client": {client},
"X-FluidDB-Client-URL": {clientURL},
"X-FluidDB-Version": {version},
"Authorization": {"Basic " + encodedUsernameAndPassword(user, pwd)},
}
} else {
req.Header = map[string][]string{
"Content-Type": {bodyType},
"Transfer-Encoding": {"chunked"},
"User-Agent": {agent},
"X-FluidDB-Client": {client},
"X-FluidDB-Client-URL": {clientURL},
"X-FluidDB-Version": {version},
}
}
req.URL, err = http.ParseURL(url)
if err != nil {
return nil, err
}
return send(&req)
}