func (session *session) SetAuthorization(uri *url.URL, domain []string, auth string) {
session.Lock()
defer session.Unlock()
if domain == nil || len(domain) == 0 {
root := &url.URL{
Scheme: uri.Scheme,
Opaque: uri.Opaque,
User: uri.User,
Host: uri.Host,
Path: "/",
RawQuery: "",
Fragment: "",
}
session.authcache.Set(root, auth)
return
}
for _, s := range domain {
ref, err := url.Parse(s)
if err == nil {
session.authcache.Set(uri.ResolveReference(ref), auth)
}
}
}
// HandleError tries to resolve the given error, which should be a
// response to the given URL, by discharging any macaroon contained in
// it. That is, if the error cause is an *Error and its code is
// ErrDischargeRequired, then it will try to discharge
// err.Info.Macaroon. If the discharge succeeds, the discharged macaroon
// will be saved to the client's cookie jar and ResolveError will return
// nil.
//
// For any other kind of error, the original error will be returned.
func (c *Client) HandleError(reqURL *url.URL, err error) error {
respErr, ok := errgo.Cause(err).(*Error)
if !ok {
return err
}
if respErr.Code != ErrDischargeRequired {
return respErr
}
if respErr.Info == nil || respErr.Info.Macaroon == nil {
return errgo.New("no macaroon found in discharge-required response")
}
mac := respErr.Info.Macaroon
macaroons, err := bakery.DischargeAllWithKey(mac, c.dischargeAcquirer().AcquireDischarge, c.Key)
if err != nil {
return errgo.Mask(err, errgo.Any)
}
var cookiePath string
if path := respErr.Info.MacaroonPath; path != "" {
relURL, err := parseURLPath(path)
if err != nil {
logger.Warningf("ignoring invalid path in discharge-required response: %v", err)
} else {
cookiePath = reqURL.ResolveReference(relURL).Path
}
}
cookie, err := NewCookie(macaroons)
if err != nil {
return errgo.Notef(err, "cannot make cookie")
}
cookie.Path = cookiePath
c.Jar.SetCookies(reqURL, []*http.Cookie{cookie})
return nil
}
func encodeURL(src []byte, baseHref string, urlString string, start int, end int) []byte {
relURL := string(src[start:end])
// keep anchor and javascript links intact
if strings.Index(relURL, "#") == 0 || strings.Index(relURL, "javascript") == 0 {
return src
}
// Check if url is relative and make it absolute
if strings.Index(relURL, "http") != 0 {
var basePath *url.URL
if baseHref == "" {
basePath, _ = url.Parse(urlString)
} else {
basePath, _ = url.Parse(baseHref)
}
relPath, err := url.Parse(relURL)
if err != nil {
return src
}
absURL := basePath.ResolveReference(relPath).String()
src = bytes.Replace(src, []byte(relURL), []byte(absURL), -1)
end = start + len(absURL)
}
newURL, _ := EncryptURL(string(src[start:end]))
return bytes.Replace(src, src[start:end], []byte(newURL), -1)
}
//Accepts several paths separated by separator and constructs the URLs
//relative to base path
func pathToUri(paths string, separator string, basePath string) (urls []url.URL, err error) {
var urlBase *url.URL
if basePath != "" {
if string(basePath[0]) != "/" {
//for windows path to build a proper url
basePath = "/" + basePath
}
urlBase, err = url.Parse("file:" + basePath)
}
if err != nil {
return nil, err
}
inputs := strings.Split(paths, ",")
for _, input := range inputs {
var urlInput *url.URL
if basePath != "" {
urlInput, err = url.Parse(filepath.ToSlash(input))
if err != nil {
return nil, err
}
urlInput = urlBase.ResolveReference(urlInput)
} else {
//TODO is opaque really apropriate?
urlInput = &url.URL{
Opaque: filepath.ToSlash(input),
}
}
urls = append(urls, *urlInput)
}
//clean
return
}
func getURL(base *url.URL, urlStr string) (*url.URL, error) {
rel, err := url.Parse(urlStr)
if err != nil {
return nil, err
}
return base.ResolveReference(rel), nil
}
func (s *Scenario) GetAsset(w *Worker, base *url.URL, node xml.Node, attr string) error {
path, err := url.Parse(node.Attr(attr))
if err != nil {
return w.Fail(nil, err)
}
requestURI := base.ResolveReference(path)
req, res, err := w.SimpleGet(requestURI.String())
if err != nil {
return w.Fail(req, err)
}
if res.StatusCode != 200 {
return w.Fail(res.Request, fmt.Errorf("Response code should be %d, got %d", 200, res.StatusCode))
}
md5sum := calcMD5(res.Body)
defer res.Body.Close()
if expectedMD5, ok := s.ExpectedAssets[requestURI.RequestURI()]; ok {
if md5sum == expectedMD5 {
w.Success(StaticFileScore)
} else {
return w.Fail(res.Request, fmt.Errorf("Expected MD5 checksum is miss match %s, got %s", expectedMD5, md5sum))
}
}
return nil
}
func getFileInfo(base *url.URL, reader io.Reader) (*result, error) {
h := sha1.New()
buf := make([]byte, 1024)
tee := io.TeeReader(reader, h)
res := result{}
config, format, err := image.DecodeConfig(tee)
if err != nil {
return nil, err
}
res.Width = config.Width
res.Height = config.Height
for ; err != io.EOF; _, err = tee.Read(buf) {
if err != nil {
return nil, err
}
}
hash := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
hashURL, _ := url.Parse(hash + "." + format)
res.URI = base.ResolveReference(hashURL).String()
res.UUID = uuid.NewV5(uuid.NamespaceURL, res.URI)
return &res, nil
}
// newRequest is a helper function to generate an http.Request and automagically
// json encode a body and resolve the given path.
func newRequest(baseURL *url.URL, method, path string, body interface{}) (*http.Request, error) {
rel, err := url.Parse(path)
if err != nil {
return nil, err
}
uri := baseURL.ResolveReference(rel)
var buf io.ReadWriter
if body != nil {
buf = new(bytes.Buffer)
err := json.NewEncoder(buf).Encode(body)
if err != nil {
return nil, err
}
}
req, err := http.NewRequest(method, uri.String(), buf)
if err != nil {
return nil, err
}
req.Header.Add(headerAccept, acceptType)
req.Header.Add(headerContentType, contentType)
return req, nil
}
func cleanHref(pageUrl *url.URL, href string) string {
if href == "" {
return ""
}
href = strings.TrimSpace(href)
parsedHref, err := url.Parse(href)
if err != nil {
return ""
}
// convert www subdomain to host without www,
// to avoid duplicate path uri
if strings.Contains(parsedHref.Host, "www") {
// host is www.example.com
s := strings.Split(parsedHref.Host, "www.")
// s is [ example.com]
parsedHref.Host = strings.TrimSpace(s[1])
// host is now example.com
}
// turn relative href uri into absolute,
// using pageUrl as base
if !parsedHref.IsAbs() {
parsedHref = pageUrl.ResolveReference(parsedHref)
}
// add / to end of href path for consistency
if parsedHref.Path != "" && parsedHref.Path != "/" {
lastChar := parsedHref.Path[len(parsedHref.Path)-1:]
if lastChar != "/" {
parsedHref.Path = parsedHref.Path + "/"
}
}
return parsedHref.String()
}
func (f *Fetcher) Fetch(u *url.URL) (urls_obj []*url.URL, err error) {
html, err := f.get(u)
if err != nil {
return nil, err
}
urls, err := f.Picker.Pick(html)
if err != nil {
return nil, err
}
//update non host url
for _, v := range urls {
u_child, err := url.Parse(v)
//skip on invalid url
if err != nil {
continue
}
urls_obj = append(urls_obj, u.ResolveReference(u_child))
}
return urls_obj, nil
}
func ToAbsolute(base *url.URL, rawUrl string) string {
url, err := url.Parse(rawUrl)
if err != nil {
return rawUrl
}
return base.ResolveReference(url).String()
}
func getAbsoluteURL(pageURL, targetURL *url.URL) (*url.URL, error) {
t, err := url.Parse(targetURL.String())
if err != nil {
return nil, err
}
return pageURL.ResolveReference(t), nil
}
func (t *AbsURL) absUrlify(tr *htmltran.Transformer, selectors ...elattr) (err error) {
var baseURL, inURL *url.URL
if baseURL, err = url.Parse(t.BaseURL); err != nil {
return
}
replace := func(in string) string {
if inURL, err = url.Parse(in); err != nil {
return in + "?"
}
if fragmentOnly(inURL) {
return in
}
return baseURL.ResolveReference(inURL).String()
}
for _, el := range selectors {
if err = tr.Apply(htmltran.TransformAttrib(el.attr, replace), el.tag); err != nil {
return
}
}
return
}
// newRequest creates an API request. A relative URL can be provided in urlStr,
// in which case it is resolved relative to base URL.
// Relative URLs should always be specified without a preceding slash. If
// specified, the value pointed to by body is JSON encoded and included as the
// request body.
func (c *Client) newRequest(base *url.URL, method, urlStr string, body interface{}) (*http.Request, error) {
rel, err := url.Parse(urlStr)
if err != nil {
return nil, err
}
u := base.ResolveReference(rel)
bodyReader, ok := body.(io.Reader)
if !ok && body != nil {
buf := &bytes.Buffer{}
err := json.NewEncoder(buf).Encode(body)
if err != nil {
return nil, err
}
bodyReader = buf
}
req, err := http.NewRequest(method, u.String(), bodyReader)
if err != nil {
return nil, err
}
// req.Header.Add("Accept", mediaTypeV3)
if c.UserAgent != "" {
req.Header.Add("User-Agent", c.UserAgent)
}
return req, nil
}
func resolveUrl(base *url.URL, href string) (*url.URL, error) {
u, err := url.Parse(href)
if err != nil {
return nil, err
}
return base.ResolveReference(u), nil
}
func resolveSchemaURIWithBase(base *url.URL, uri string) string {
u, err := url.Parse(uri)
if err != nil {
log.Fatalf("Expected URL to parse: %q, got error: %v", uri, err)
}
return base.ResolveReference(u).String()
}
func toAbsURL(baseurl *url.URL, weburl string) (*url.URL, error) {
relurl, err := url.Parse(weburl)
if err != nil {
return nil, err
}
absurl := baseurl.ResolveReference(relurl)
return absurl, nil
}
func ToAbsUrl(baseurl *url.URL, weburl string) string {
relurl, err := url.Parse(weburl)
if err != nil {
return ""
}
absurl := baseurl.ResolveReference(relurl)
return absurl.String()
}
func resolveUrl(root *url.URL, documentURL string) (gojsonreference.JsonReference, error) {
ref, err := gojsonreference.NewJsonReference(documentURL)
if err != nil {
return ref, err
}
resolvedURL := root.ResolveReference(ref.GetUrl())
return gojsonreference.NewJsonReference(resolvedURL.String())
}
func toAbsURL(base_url *url.URL, path string) (string, error) {
relurl, err := url.Parse(path)
if err != nil {
return "", err
}
absurl := base_url.ResolveReference(relurl)
return absurl.String(), nil
}
// resolveURL converts all input URLs into absolute URLs.
func resolveURL(s string, link *url.URL) (string, error) {
// x may or may not be an absolute URL.
x, err := url.Parse(s)
if err != nil {
return "", err
}
// y is guaranteed to be an absolute URL.
y := link.ResolveReference(x)
return y.String(), nil
}
// getSitemap returns a sitemap containing both templated pages
// and schedule session details.
func getSitemap(c context.Context, baseURL *url.URL) (*sitemap, error) {
items := make([]*sitemapItem, 0)
// templated pages
root := filepath.Join(config.Dir, templatesDir)
err := filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
ext := filepath.Ext(p)
if p == root || fi.IsDir() || ext != ".html" {
return nil
}
name := p[len(root)+1 : len(p)-len(ext)]
for _, s := range skipSitemap {
if strings.HasPrefix(name, s) {
return nil
}
}
freq := "weekly"
if name == "home" {
name = ""
freq = "daily"
}
item := &sitemapItem{
Loc: baseURL.ResolveReference(&url.URL{Path: name}).String(),
Freq: freq,
}
items = append(items, item)
return nil
})
if err != nil {
return nil, err
}
// schedule
sched, err := getLatestEventData(c, nil)
if err != nil {
return nil, err
}
mod := sched.modified.In(time.UTC)
for id, _ := range sched.Sessions {
u := baseURL.ResolveReference(&url.URL{Path: "schedule"})
u.RawQuery = url.Values{"sid": {id}}.Encode()
item := &sitemapItem{
Loc: u.String(),
Mod: &mod,
Freq: "daily",
}
items = append(items, item)
}
return &sitemap{Items: items}, nil
}
func buildRedirectURL(redirectTemplate string, baseRequestURL, requestURL *url.URL) (*url.URL, error) {
if baseRequestURL != nil {
requestURL = baseRequestURL.ResolveReference(requestURL)
}
redirectURL, err := url.Parse(redirectTemplate)
if err != nil {
return nil, err
}
redirectURL.RawQuery = strings.Replace(redirectURL.RawQuery, QueryToken, requestURL.RawQuery, -1)
redirectURL.RawQuery = strings.Replace(redirectURL.RawQuery, URLToken, url.QueryEscape(requestURL.String()), -1)
return redirectURL, nil
}
// resolves a ref into an absolute, crawlable url
func resolveAbsoluteUrl(pageUrl *url.URL, ref string) (*url.URL, error) {
refUrl, err := url.Parse(ref)
if err != nil {
return nil, err
}
refUrl = pageUrl.ResolveReference(refUrl)
// fragments are not significant for crawling
refUrl.Fragment = ""
return refUrl, nil
}
// parseCmd returns cmd by resolving a url
func parseCmd(s string, lnk *url.URL) (*Cmd, error) {
href, err := url.Parse(s)
if err != nil {
cognilog.Log("red", err)
return nil, err
}
url := lnk.ResolveReference(href)
return &Cmd{
U: url,
M: "GET",
}, nil
}
// scrapeLoginPage simulates a user visiting the given web
// page. It gets the login page, then does a POST with
// the appropriate form parameters.
func scrapeLoginPage(loginURL *url.URL) error {
log.Printf("scraping login page")
// Get the page.
client := httpbakery.DefaultHTTPClient
log.Printf("scrape: getting %s", loginURL)
resp, err := client.Get(loginURL.String())
if err != nil {
return errgo.Mask(err)
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errgo.Notef(err, "cannot read body")
}
m := actionPat.FindSubmatch(data)
if m == nil {
return errgo.New("cannot find match for action")
}
action := string(m[1])
m = waitIdPat.FindSubmatch(data)
if m == nil {
return errgo.New("cannot find match for waitid")
}
waitId := string(m[1])
actionURL, err := url.Parse(action)
if err != nil {
return errgo.Notef(err, "cannot parse action URL %q", action)
}
// Now simulate the user clicking on "Log in".
postURL := loginURL.ResolveReference(actionURL)
log.Printf("posting to %s (waitId %s)", postURL, waitId)
postResp, err := client.PostForm(postURL.String(), url.Values{
"user": {"root"},
"password": {"superman"},
"waitid": {waitId},
})
if err != nil {
return errgo.Notef(err, "cannot post")
}
defer postResp.Body.Close()
if postResp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(postResp.Body)
if err != nil {
return errgo.Notef(err, "cannot read body")
}
return errgo.Newf("post failed with status %s (body %q)", postResp.Status, body)
}
return nil
}
func visitUrl(urlToVisit *url.URL, wg *sync.WaitGroup, visitedUrls *visitedUrlsStruct) {
defer wg.Done()
if visitedUrls.hasKey(urlToVisit.String()) {
return
}
visitedUrls.addKey(urlToVisit.String())
//fetch
nbWorker <- true
defer func() { <-nbWorker }()
resp, err := http.Get(urlToVisit.String())
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
fmt.Println(err)
return
}
trs := doc.Find("table tr")
trs.Each(func(index int, node *goquery.Selection) {
tds := node.Find("td a")
if tds.Size() > 0 {
val := tds.Get(0).Attr[0]
new_path, err := url.Parse(val.Val)
if err != nil {
panic(err)
}
recomposed_url := urlToVisit.ResolveReference(new_path)
if strings.HasSuffix(recomposed_url.Path, "/") {
//craw children
wg.Add(1)
go visitUrl(recomposed_url, wg, visitedUrls)
} else {
*(visitedUrls.results) <- recomposed_url.String()
}
}
})
return
}
func absURL(req *http.Request, path string) string {
server := url.URL{
Host: req.Host,
}
base := server.ResolveReference(req.URL)
u, err := url.Parse(path)
if err != nil {
log.Panic(err)
}
return base.ResolveReference(u).String()
}
func GetRobotsForURL(target *url.URL, factory client.ClientFactory) (*RobotsData, error) {
client := factory.Get()
ref, _ := url.Parse("/robots.txt")
robotsURL := target.ResolveReference(ref)
resp, err := client.RequestURL(robotsURL)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return ParseRobotsTxt(body)
}
func (s *Server) itemByDir(dir string, baseUrl *url.URL) (*PodcastItem, error) {
_, mp3Stat, err := s.mp3Path(dir)
if err != nil {
return nil, err
}
xmlPath, _, err := s.xmlPath(dir)
if err != nil {
return nil, err
}
xmlFile, err := os.Open(xmlPath)
if err != nil {
return nil, err
}
defer xmlFile.Close()
dec := xml.NewDecoder(xmlFile)
var prog RadikoProg
if err := dec.Decode(&prog); err != nil {
return nil, err
}
u, err := url.Parse("/podcast/" + dir + ".mp3")
if err != nil {
return nil, err
}
ft, _ := prog.FtTime()
var item PodcastItem
item.Title = fmt.Sprintf("%s (%s)", prog.Title, ft)
item.ITunesAuthor = prog.Pfm
item.ITunesSummary = prog.Info
item.Enclosure.Url = baseUrl.ResolveReference(u).String()
item.Enclosure.Type = "audio/mpeg"
item.Enclosure.Length = int(mp3Stat.Size())
item.PubDate = PubDate{mp3Stat.ModTime()}
return &item, nil
}