func doGet(c *cli.Context) {
argURL := c.Args().Get(0)
doUpdate := c.Bool("update")
if argURL == "" {
cli.ShowCommandHelp(c, "get")
os.Exit(1)
}
url, err := url.Parse(argURL)
utils.DieIf(err)
if !url.IsAbs() {
url.Scheme = "https"
url.Host = "github.com"
if url.Path[0] != '/' {
url.Path = "/" + url.Path
}
}
remote, err := NewRemoteRepository(url)
utils.DieIf(err)
if remote.IsValid() == false {
utils.Log("error", fmt.Sprintf("Not a valid repository: %s", url))
os.Exit(1)
}
getRemoteRepository(remote, doUpdate)
}
func (markdown Markdown) Extract(creativeWork *schema.CreativeWork, path string) error {
markdownContent, err := ioutil.ReadFile(path)
if nil != err {
return err
}
unsafe := blackfriday.MarkdownCommon(markdownContent)
p := bluemonday.UGCPolicy()
p.RequireNoFollowOnLinks(false)
p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
html := p.SanitizeBytes(unsafe)
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(html))
if nil != err {
return err
}
doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
link, _ := s.Attr("href")
url, _ := url.Parse(link)
if !url.IsAbs() {
s.SetAttr("href", strings.Replace(link, ".md", ".jsonld", 1))
}
})
creativeWork.Name = doc.Find("h1").Text()
creativeWork.Text, err = doc.Find("body").Html()
if nil != err {
return err
}
return nil
}
func formatURL(ref string) (*url.URL, error) {
if !hasSchemePattern.MatchString(ref) && scpLikeURLPattern.MatchString(ref) {
matched := scpLikeURLPattern.FindStringSubmatch(ref)
user := matched[1]
host := matched[2]
path := matched[3]
ref = fmt.Sprintf("ssh://%s%s/%s", user, host, path)
}
url, err := url.Parse(ref)
if err != nil {
return url, err
}
if !url.IsAbs() {
if !strings.Contains(url.Path, "/") {
url.Path = url.Path + "/" + url.Path
}
url.Scheme = "https"
url.Host = "github.com"
if url.Path[0] != '/' {
url.Path = "/" + url.Path
}
}
return url, nil
}
func (markdown Markdown) Extract(creativeWork *schema.CreativeWork, path string) error {
markdownContent, err := ioutil.ReadFile(path)
if nil != err {
return err
}
html := blackfriday.MarkdownCommon(markdownContent)
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(html))
if nil != err {
return err
}
doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
link, _ := s.Attr("href")
url, _ := url.Parse(link)
if !url.IsAbs() && strings.HasSuffix(link, ".md") {
s.SetAttr("href", fmt.Sprint(link[:len(link)-3], ".jsonld"))
}
})
creativeWork.Name = doc.Find("h1").Text()
creativeWork.Text, err = doc.Find("body").Html()
if nil != err {
return err
}
return nil
}
func resolveCacheKey(id string) (string, error) {
url, err := url.Parse(id)
if err == nil && url.IsAbs() {
url.Fragment = ""
return strings.TrimSuffix(url.String(), "/"), nil
}
return "", fmt.Errorf("failed to resolve cache key for %s", id)
}
// IsAbsURL determines whether the given path points to an absolute URL.
func IsAbsURL(path string) bool {
url, err := url.Parse(path)
if err != nil {
return false
}
return url.IsAbs() || strings.HasPrefix(path, "//")
}
func (c *K8sConnector) SetBaseURL(u string) error {
url, error := url.Parse(u)
if error != nil {
return error
}
if !url.IsAbs() {
return errors.New("k8sclient: Kubernetes endpoint url must be an absolute URL")
}
c.baseURL = url.String()
return nil
}
func (t *targets) Set(str string) error {
url, err := url.Parse(str)
if err != nil {
return err
}
if !url.IsAbs() {
return errURLNotAbsolute
}
if !pingers.CanHandle(url) {
return errNoPinger
}
*t = append(*t, url)
return nil
}
// NewUrl sets the client's new URL (yes, it mutates).
// If NewUrl is a relative URL then it will be based
// on the previous value of the URL that the RestClient had.
func (rc *RestClient) NewUrl(dest string) error {
url, err := url.Parse(dest)
if err != nil {
return err
}
if rc.url == nil {
if !url.IsAbs() {
return errors.New("Expected absolute URL.")
} else {
rc.url = url
}
} else {
NewUrl := rc.url.ResolveReference(url)
log.Printf("Getting %s, resolved reference from %s to %s: %s\n", dest, rc.url, url, NewUrl)
rc.url = NewUrl
}
return nil
}
func (proxy *HttpProxy) urlProxyHandler(responseWriter http.ResponseWriter, request *http.Request) {
var client *http.Client
var originUrl string
var err error
// Request URL should be "/tunneled/<origin URL>" or "/direct/<origin URL>" and the
// origin URL must be URL encoded.
switch {
case strings.HasPrefix(request.URL.RawPath, URL_PROXY_TUNNELED_REQUEST_PATH):
originUrl, err = url.QueryUnescape(request.URL.RawPath[len(URL_PROXY_TUNNELED_REQUEST_PATH):])
client = proxy.urlProxyTunneledClient
case strings.HasPrefix(request.URL.RawPath, URL_PROXY_DIRECT_REQUEST_PATH):
originUrl, err = url.QueryUnescape(request.URL.RawPath[len(URL_PROXY_DIRECT_REQUEST_PATH):])
client = proxy.urlProxyDirectClient
default:
err = errors.New("missing origin URL")
}
if err != nil {
NoticeAlert("%s", common.ContextError(FilterUrlError(err)))
forceClose(responseWriter)
return
}
// Origin URL must be well-formed, absolute, and have a scheme of "http" or "https"
url, err := url.ParseRequestURI(originUrl)
if err != nil {
NoticeAlert("%s", common.ContextError(FilterUrlError(err)))
forceClose(responseWriter)
return
}
if !url.IsAbs() || (url.Scheme != "http" && url.Scheme != "https") {
NoticeAlert("invalid origin URL")
forceClose(responseWriter)
return
}
// Transform received request to directly reference the origin URL
request.Host = url.Host
request.URL = url
relayHttpRequest(client, nil, request, responseWriter)
}
// AbsURL creates a absolute URL from the relative path given and the BaseURL set in config.
func AbsURL(in string, addLanguage bool) string {
url, err := url.Parse(in)
if err != nil {
return in
}
if url.IsAbs() || strings.HasPrefix(in, "//") {
return in
}
baseURL := viper.GetString("BaseURL")
if strings.HasPrefix(in, "/") {
p, err := url.Parse(baseURL)
if err != nil {
panic(err)
}
p.Path = ""
baseURL = p.String()
}
if addLanguage {
prefix := getLanguagePrefix()
if prefix != "" {
hasPrefix := false
// avoid adding language prefix if already present
if strings.HasPrefix(in, "/") {
hasPrefix = strings.HasPrefix(in[1:], prefix)
} else {
hasPrefix = strings.HasPrefix(in, prefix)
}
if !hasPrefix {
addSlash := in == "" || strings.HasSuffix(in, "/")
in = path.Join(prefix, in)
if addSlash {
in += "/"
}
}
}
}
return MakePermalink(baseURL, in).String()
}
// Check to see if the remote passed to this structure is valid.
// Currently, validity consists of:
// remote.Urlbase being a valid URL without any embedded user info.
// remote.Ulrbase starting with git, http, https, or ssh.
// remote.Name passing validRemoteName
// remote.Priority being between 1 and 100
func ValidateRemote(remote *Remote) bool {
url, err := url.Parse(remote.Urlbase)
if err != nil {
log.Printf("%s is not a URL!\n", remote.Urlbase)
return false
} else if !url.IsAbs() {
log.Printf("%s is not an absolute URL!\n", remote.Urlbase)
return false
}
switch url.Scheme {
case "git":
fallthrough
case "http":
fallthrough
case "https":
if url.User != nil {
log.Printf("Please don't embed userinfo in your http(s) or git URL!\n")
log.Printf("Instead, modify your .netrc to include it for %s\n", url.Host)
log.Printf("Example:\n")
log.Printf(" machine %s login <username> password <password>\n", url.Host)
return false
}
case "ssh":
if url.User == nil {
log.Printf("%s does not include an embedded username!", remote.Urlbase)
return false
}
default:
log.Printf("URL scheme %s is not supported by the dev tool for now.", url.Scheme)
return false
}
if remote.Name == "" {
remote.Name = filepath.Base(url.Path)
}
if !validRemoteName(remote.Name) {
return false
}
if remote.Priority < 1 || remote.Priority > 100 {
log.Printf("Priority must be a number between 1 and 100 (currently %d)!\n", remote.Priority)
return false
}
return true
}
// AbsURL creates a absolute URL from the relative path given and the BaseURL set in config.
func AbsURL(path string) string {
url, err := url.Parse(path)
if err != nil {
return path
}
if url.IsAbs() || strings.HasPrefix(path, "//") {
return path
}
baseURL := viper.GetString("BaseURL")
if strings.HasPrefix(path, "/") {
p, err := url.Parse(baseURL)
if err != nil {
panic(err)
}
p.Path = ""
baseURL = p.String()
}
return MakePermalink(baseURL, path).String()
}
func (p BBSPresence) Validate() error {
var validationError ValidationError
if p.ID == "" {
validationError = validationError.Append(ErrInvalidField{Field: "id"})
}
if p.URL == "" {
validationError = validationError.Append(ErrInvalidField{Field: "url"})
}
url, err := url.Parse(p.URL)
if err != nil || !url.IsAbs() {
validationError = validationError.Append(ErrInvalidField{Field: "url"})
}
if !validationError.Empty() {
return validationError
}
return nil
}
func findImage(blog *Blog, post *Post, name string) (uri string, err error, cfg image.Config) {
// If it's an absolute URL, pass it through - but we don't know the size.
if url, urlerr := url.Parse(name); urlerr == nil && url.IsAbs() {
uri = name
return
}
// Else we assume it's a regular path, which has to be relative.
if path.IsAbs(name) {
err = fmt.Errorf("%q: image %q needs to be either an absolute URL or a relative path.", post.Id, name)
return
}
// If the path name contains a slash, assume it's a full path
// in the content dir.
if strings.IndexRune(name, '/') != -1 {
var found bool
filepath := filepath.Join(blog.PostDir, name)
uri = name
if found, err, cfg = tryAddImage(blog, post, filepath, uri); found {
return
}
} else {
// Search first in asset dirs for this post, then parent posts
for p := post; p != nil; p = p.Parent {
var found bool
filepath := filepath.Join(blog.PostDir, p.AssetPath(), name)
uri = path.Join(p.AssetPath(), name)
if found, err, cfg = tryAddImage(blog, post, filepath, uri); found {
return
}
}
}
err = fmt.Errorf("%q: Image %q not found.", post.Id, name)
return
}