Exemple #1
0
func (session *session) SetAuthorization(uri *url.URL, domain []string, auth string) {
	session.Lock()
	defer session.Unlock()

	if domain == nil || len(domain) == 0 {
		root := &url.URL{
			Scheme:   uri.Scheme,
			Opaque:   uri.Opaque,
			User:     uri.User,
			Host:     uri.Host,
			Path:     "/",
			RawQuery: "",
			Fragment: "",
		}
		session.authcache.Set(root, auth)
		return
	}

	for _, s := range domain {
		ref, err := url.Parse(s)
		if err == nil {
			session.authcache.Set(uri.ResolveReference(ref), auth)
		}
	}
}
Exemple #2
0
// HandleError tries to resolve the given error, which should be a
// response to the given URL, by discharging any macaroon contained in
// it. That is, if the error cause is an *Error and its code is
// ErrDischargeRequired, then it will try to discharge
// err.Info.Macaroon. If the discharge succeeds, the discharged macaroon
// will be saved to the client's cookie jar and ResolveError will return
// nil.
//
// For any other kind of error, the original error will be returned.
func (c *Client) HandleError(reqURL *url.URL, err error) error {
	respErr, ok := errgo.Cause(err).(*Error)
	if !ok {
		return err
	}
	if respErr.Code != ErrDischargeRequired {
		return respErr
	}
	if respErr.Info == nil || respErr.Info.Macaroon == nil {
		return errgo.New("no macaroon found in discharge-required response")
	}
	mac := respErr.Info.Macaroon
	macaroons, err := bakery.DischargeAllWithKey(mac, c.dischargeAcquirer().AcquireDischarge, c.Key)
	if err != nil {
		return errgo.Mask(err, errgo.Any)
	}
	var cookiePath string
	if path := respErr.Info.MacaroonPath; path != "" {
		relURL, err := parseURLPath(path)
		if err != nil {
			logger.Warningf("ignoring invalid path in discharge-required response: %v", err)
		} else {
			cookiePath = reqURL.ResolveReference(relURL).Path
		}
	}
	cookie, err := NewCookie(macaroons)
	if err != nil {
		return errgo.Notef(err, "cannot make cookie")
	}
	cookie.Path = cookiePath
	c.Jar.SetCookies(reqURL, []*http.Cookie{cookie})
	return nil
}
Exemple #3
0
func encodeURL(src []byte, baseHref string, urlString string, start int, end int) []byte {
	relURL := string(src[start:end])
	// keep anchor and javascript links intact
	if strings.Index(relURL, "#") == 0 || strings.Index(relURL, "javascript") == 0 {
		return src
	}
	// Check if url is relative and make it absolute
	if strings.Index(relURL, "http") != 0 {
		var basePath *url.URL
		if baseHref == "" {
			basePath, _ = url.Parse(urlString)
		} else {
			basePath, _ = url.Parse(baseHref)
		}
		relPath, err := url.Parse(relURL)
		if err != nil {
			return src
		}
		absURL := basePath.ResolveReference(relPath).String()
		src = bytes.Replace(src, []byte(relURL), []byte(absURL), -1)
		end = start + len(absURL)
	}
	newURL, _ := EncryptURL(string(src[start:end]))
	return bytes.Replace(src, src[start:end], []byte(newURL), -1)
}
Exemple #4
0
//Accepts several paths separated by separator and constructs the URLs
//relative to base path
func pathToUri(paths string, separator string, basePath string) (urls []url.URL, err error) {
	var urlBase *url.URL

	if basePath != "" {
		if string(basePath[0]) != "/" {
			//for windows path to build a proper url
			basePath = "/" + basePath
		}
		urlBase, err = url.Parse("file:" + basePath)
	}
	if err != nil {
		return nil, err
	}
	inputs := strings.Split(paths, ",")
	for _, input := range inputs {
		var urlInput *url.URL
		if basePath != "" {
			urlInput, err = url.Parse(filepath.ToSlash(input))
			if err != nil {
				return nil, err
			}
			urlInput = urlBase.ResolveReference(urlInput)
		} else {
			//TODO is opaque really apropriate?
			urlInput = &url.URL{
				Opaque: filepath.ToSlash(input),
			}
		}
		urls = append(urls, *urlInput)
	}
	//clean
	return
}
Exemple #5
0
func getURL(base *url.URL, urlStr string) (*url.URL, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	return base.ResolveReference(rel), nil
}
Exemple #6
0
func (s *Scenario) GetAsset(w *Worker, base *url.URL, node xml.Node, attr string) error {
	path, err := url.Parse(node.Attr(attr))

	if err != nil {
		return w.Fail(nil, err)
	}

	requestURI := base.ResolveReference(path)

	req, res, err := w.SimpleGet(requestURI.String())

	if err != nil {
		return w.Fail(req, err)
	}

	if res.StatusCode != 200 {
		return w.Fail(res.Request, fmt.Errorf("Response code should be %d, got %d", 200, res.StatusCode))
	}

	md5sum := calcMD5(res.Body)
	defer res.Body.Close()

	if expectedMD5, ok := s.ExpectedAssets[requestURI.RequestURI()]; ok {
		if md5sum == expectedMD5 {
			w.Success(StaticFileScore)
		} else {
			return w.Fail(res.Request, fmt.Errorf("Expected MD5 checksum is miss match %s, got %s", expectedMD5, md5sum))
		}
	}

	return nil
}
Exemple #7
0
func getFileInfo(base *url.URL, reader io.Reader) (*result, error) {
	h := sha1.New()
	buf := make([]byte, 1024)

	tee := io.TeeReader(reader, h)
	res := result{}

	config, format, err := image.DecodeConfig(tee)
	if err != nil {
		return nil, err
	}
	res.Width = config.Width
	res.Height = config.Height

	for ; err != io.EOF; _, err = tee.Read(buf) {
		if err != nil {
			return nil, err
		}
	}

	hash := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
	hashURL, _ := url.Parse(hash + "." + format)
	res.URI = base.ResolveReference(hashURL).String()
	res.UUID = uuid.NewV5(uuid.NamespaceURL, res.URI)
	return &res, nil
}
Exemple #8
0
// newRequest is a helper function to generate an http.Request and automagically
// json encode a body and resolve the given path.
func newRequest(baseURL *url.URL, method, path string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	uri := baseURL.ResolveReference(rel)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, uri.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add(headerAccept, acceptType)
	req.Header.Add(headerContentType, contentType)

	return req, nil
}
func cleanHref(pageUrl *url.URL, href string) string {
	if href == "" {
		return ""
	}
	href = strings.TrimSpace(href)

	parsedHref, err := url.Parse(href)
	if err != nil {
		return ""
	}
	// convert www subdomain to host without www,
	// to avoid duplicate path uri
	if strings.Contains(parsedHref.Host, "www") {
		// host is www.example.com
		s := strings.Split(parsedHref.Host, "www.")
		// s is [ example.com]
		parsedHref.Host = strings.TrimSpace(s[1])
		// host is now example.com
	}
	// turn relative href uri into absolute,
	// using pageUrl as base
	if !parsedHref.IsAbs() {
		parsedHref = pageUrl.ResolveReference(parsedHref)
	}
	// add / to end of href path for consistency
	if parsedHref.Path != "" && parsedHref.Path != "/" {
		lastChar := parsedHref.Path[len(parsedHref.Path)-1:]
		if lastChar != "/" {
			parsedHref.Path = parsedHref.Path + "/"
		}
	}

	return parsedHref.String()
}
Exemple #10
0
func (f *Fetcher) Fetch(u *url.URL) (urls_obj []*url.URL, err error) {
	html, err := f.get(u)

	if err != nil {
		return nil, err
	}

	urls, err := f.Picker.Pick(html)

	if err != nil {
		return nil, err
	}

	//update non host url
	for _, v := range urls {
		u_child, err := url.Parse(v)

		//skip on invalid url
		if err != nil {
			continue
		}

		urls_obj = append(urls_obj, u.ResolveReference(u_child))
	}

	return urls_obj, nil
}
Exemple #11
0
func ToAbsolute(base *url.URL, rawUrl string) string {
	url, err := url.Parse(rawUrl)
	if err != nil {
		return rawUrl
	}
	return base.ResolveReference(url).String()
}
Exemple #12
0
func getAbsoluteURL(pageURL, targetURL *url.URL) (*url.URL, error) {
	t, err := url.Parse(targetURL.String())
	if err != nil {
		return nil, err
	}
	return pageURL.ResolveReference(t), nil
}
Exemple #13
0
func (t *AbsURL) absUrlify(tr *htmltran.Transformer, selectors ...elattr) (err error) {
	var baseURL, inURL *url.URL

	if baseURL, err = url.Parse(t.BaseURL); err != nil {
		return
	}

	replace := func(in string) string {
		if inURL, err = url.Parse(in); err != nil {
			return in + "?"
		}
		if fragmentOnly(inURL) {
			return in
		}
		return baseURL.ResolveReference(inURL).String()
	}

	for _, el := range selectors {
		if err = tr.Apply(htmltran.TransformAttrib(el.attr, replace), el.tag); err != nil {
			return
		}
	}

	return
}
Exemple #14
0
// newRequest creates an API request. A relative URL can be provided in urlStr,
// in which case it is resolved relative to base URL.
// Relative URLs should always be specified without a preceding slash. If
// specified, the value pointed to by body is JSON encoded and included as the
// request body.
func (c *Client) newRequest(base *url.URL, method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	u := base.ResolveReference(rel)

	bodyReader, ok := body.(io.Reader)
	if !ok && body != nil {
		buf := &bytes.Buffer{}
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
		bodyReader = buf
	}

	req, err := http.NewRequest(method, u.String(), bodyReader)
	if err != nil {
		return nil, err
	}

	//	req.Header.Add("Accept", mediaTypeV3)
	if c.UserAgent != "" {
		req.Header.Add("User-Agent", c.UserAgent)
	}
	return req, nil
}
Exemple #15
0
func resolveUrl(base *url.URL, href string) (*url.URL, error) {
	u, err := url.Parse(href)
	if err != nil {
		return nil, err
	}

	return base.ResolveReference(u), nil
}
Exemple #16
0
func resolveSchemaURIWithBase(base *url.URL, uri string) string {

	u, err := url.Parse(uri)
	if err != nil {
		log.Fatalf("Expected URL to parse: %q, got error: %v", uri, err)
	}
	return base.ResolveReference(u).String()
}
Exemple #17
0
func toAbsURL(baseurl *url.URL, weburl string) (*url.URL, error) {
	relurl, err := url.Parse(weburl)
	if err != nil {
		return nil, err
	}
	absurl := baseurl.ResolveReference(relurl)
	return absurl, nil
}
Exemple #18
0
func ToAbsUrl(baseurl *url.URL, weburl string) string {
	relurl, err := url.Parse(weburl)
	if err != nil {
		return ""
	}
	absurl := baseurl.ResolveReference(relurl)
	return absurl.String()
}
Exemple #19
0
func resolveUrl(root *url.URL, documentURL string) (gojsonreference.JsonReference, error) {
	ref, err := gojsonreference.NewJsonReference(documentURL)
	if err != nil {
		return ref, err
	}
	resolvedURL := root.ResolveReference(ref.GetUrl())

	return gojsonreference.NewJsonReference(resolvedURL.String())
}
Exemple #20
0
func toAbsURL(base_url *url.URL, path string) (string, error) {
	relurl, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	absurl := base_url.ResolveReference(relurl)

	return absurl.String(), nil
}
Exemple #21
0
// resolveURL converts all input URLs into absolute URLs.
func resolveURL(s string, link *url.URL) (string, error) {
	// x may or may not be an absolute URL.
	x, err := url.Parse(s)
	if err != nil {
		return "", err
	}
	// y is guaranteed to be an absolute URL.
	y := link.ResolveReference(x)
	return y.String(), nil
}
Exemple #22
0
// getSitemap returns a sitemap containing both templated pages
// and schedule session details.
func getSitemap(c context.Context, baseURL *url.URL) (*sitemap, error) {
	items := make([]*sitemapItem, 0)

	// templated pages
	root := filepath.Join(config.Dir, templatesDir)
	err := filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		ext := filepath.Ext(p)
		if p == root || fi.IsDir() || ext != ".html" {
			return nil
		}
		name := p[len(root)+1 : len(p)-len(ext)]
		for _, s := range skipSitemap {
			if strings.HasPrefix(name, s) {
				return nil
			}
		}
		freq := "weekly"
		if name == "home" {
			name = ""
			freq = "daily"
		}
		item := &sitemapItem{
			Loc:  baseURL.ResolveReference(&url.URL{Path: name}).String(),
			Freq: freq,
		}
		items = append(items, item)
		return nil
	})
	if err != nil {
		return nil, err
	}

	// schedule
	sched, err := getLatestEventData(c, nil)
	if err != nil {
		return nil, err
	}
	mod := sched.modified.In(time.UTC)
	for id, _ := range sched.Sessions {
		u := baseURL.ResolveReference(&url.URL{Path: "schedule"})
		u.RawQuery = url.Values{"sid": {id}}.Encode()
		item := &sitemapItem{
			Loc:  u.String(),
			Mod:  &mod,
			Freq: "daily",
		}
		items = append(items, item)
	}

	return &sitemap{Items: items}, nil
}
Exemple #23
0
func buildRedirectURL(redirectTemplate string, baseRequestURL, requestURL *url.URL) (*url.URL, error) {
	if baseRequestURL != nil {
		requestURL = baseRequestURL.ResolveReference(requestURL)
	}
	redirectURL, err := url.Parse(redirectTemplate)
	if err != nil {
		return nil, err
	}
	redirectURL.RawQuery = strings.Replace(redirectURL.RawQuery, QueryToken, requestURL.RawQuery, -1)
	redirectURL.RawQuery = strings.Replace(redirectURL.RawQuery, URLToken, url.QueryEscape(requestURL.String()), -1)
	return redirectURL, nil
}
Exemple #24
0
// resolves a ref into an absolute, crawlable url
func resolveAbsoluteUrl(pageUrl *url.URL, ref string) (*url.URL, error) {
	refUrl, err := url.Parse(ref)
	if err != nil {
		return nil, err
	}

	refUrl = pageUrl.ResolveReference(refUrl)

	// fragments are not significant for crawling
	refUrl.Fragment = ""

	return refUrl, nil
}
Exemple #25
0
// parseCmd returns cmd by resolving a url
func parseCmd(s string, lnk *url.URL) (*Cmd, error) {
	href, err := url.Parse(s)
	if err != nil {
		cognilog.Log("red", err)
		return nil, err
	}

	url := lnk.ResolveReference(href)
	return &Cmd{
		U: url,
		M: "GET",
	}, nil
}
// scrapeLoginPage simulates a user visiting the given web
// page. It gets the login page, then does a POST with
// the appropriate form parameters.
func scrapeLoginPage(loginURL *url.URL) error {
	log.Printf("scraping login page")
	// Get the page.
	client := httpbakery.DefaultHTTPClient
	log.Printf("scrape: getting %s", loginURL)
	resp, err := client.Get(loginURL.String())
	if err != nil {
		return errgo.Mask(err)
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errgo.Notef(err, "cannot read body")
	}
	m := actionPat.FindSubmatch(data)
	if m == nil {
		return errgo.New("cannot find match for action")
	}
	action := string(m[1])
	m = waitIdPat.FindSubmatch(data)
	if m == nil {
		return errgo.New("cannot find match for waitid")
	}
	waitId := string(m[1])

	actionURL, err := url.Parse(action)
	if err != nil {
		return errgo.Notef(err, "cannot parse action URL %q", action)
	}

	// Now simulate the user clicking on "Log in".
	postURL := loginURL.ResolveReference(actionURL)
	log.Printf("posting to %s (waitId %s)", postURL, waitId)
	postResp, err := client.PostForm(postURL.String(), url.Values{
		"user":     {"root"},
		"password": {"superman"},
		"waitid":   {waitId},
	})
	if err != nil {
		return errgo.Notef(err, "cannot post")
	}
	defer postResp.Body.Close()
	if postResp.StatusCode != http.StatusOK {
		body, err := ioutil.ReadAll(postResp.Body)
		if err != nil {
			return errgo.Notef(err, "cannot read body")
		}
		return errgo.Newf("post failed with status %s (body %q)", postResp.Status, body)
	}
	return nil
}
Exemple #27
0
func visitUrl(urlToVisit *url.URL, wg *sync.WaitGroup, visitedUrls *visitedUrlsStruct) {
	defer wg.Done()
	if visitedUrls.hasKey(urlToVisit.String()) {
		return
	}

	visitedUrls.addKey(urlToVisit.String())

	//fetch
	nbWorker <- true
	defer func() { <-nbWorker }()
	resp, err := http.Get(urlToVisit.String())

	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		fmt.Println(err)
		return
	}

	trs := doc.Find("table tr")
	trs.Each(func(index int, node *goquery.Selection) {
		tds := node.Find("td a")
		if tds.Size() > 0 {

			val := tds.Get(0).Attr[0]

			new_path, err := url.Parse(val.Val)
			if err != nil {
				panic(err)
			}
			recomposed_url := urlToVisit.ResolveReference(new_path)

			if strings.HasSuffix(recomposed_url.Path, "/") {

				//craw children
				wg.Add(1)
				go visitUrl(recomposed_url, wg, visitedUrls)

			} else {
				*(visitedUrls.results) <- recomposed_url.String()
			}
		}
	})
	return
}
Exemple #28
0
func absURL(req *http.Request, path string) string {
	server := url.URL{
		Host: req.Host,
	}

	base := server.ResolveReference(req.URL)

	u, err := url.Parse(path)
	if err != nil {
		log.Panic(err)
	}

	return base.ResolveReference(u).String()
}
Exemple #29
0
func GetRobotsForURL(target *url.URL, factory client.ClientFactory) (*RobotsData, error) {
	client := factory.Get()
	ref, _ := url.Parse("/robots.txt")
	robotsURL := target.ResolveReference(ref)
	resp, err := client.RequestURL(robotsURL)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return ParseRobotsTxt(body)
}
Exemple #30
0
func (s *Server) itemByDir(dir string, baseUrl *url.URL) (*PodcastItem, error) {

	_, mp3Stat, err := s.mp3Path(dir)

	if err != nil {
		return nil, err
	}

	xmlPath, _, err := s.xmlPath(dir)

	if err != nil {
		return nil, err
	}

	xmlFile, err := os.Open(xmlPath)

	if err != nil {
		return nil, err
	}

	defer xmlFile.Close()

	dec := xml.NewDecoder(xmlFile)

	var prog RadikoProg
	if err := dec.Decode(&prog); err != nil {
		return nil, err
	}

	u, err := url.Parse("/podcast/" + dir + ".mp3")

	if err != nil {
		return nil, err
	}

	ft, _ := prog.FtTime()

	var item PodcastItem

	item.Title = fmt.Sprintf("%s (%s)", prog.Title, ft)
	item.ITunesAuthor = prog.Pfm
	item.ITunesSummary = prog.Info

	item.Enclosure.Url = baseUrl.ResolveReference(u).String()
	item.Enclosure.Type = "audio/mpeg"
	item.Enclosure.Length = int(mp3Stat.Size())
	item.PubDate = PubDate{mp3Stat.ModTime()}

	return &item, nil
}