func GetFeedUrl(u string) (string, error) {
resp, err := http.Get(u)
if err != nil {
return "", err
}
if strings.Contains(resp.Header.Get("Content-Type"), "xml") {
return u, nil
}
tree, err := html.Parse(resp.Body)
if err != nil {
return "", err
}
sel := cascadia.MustCompile("link[rel=alternate][type*=xml]")
alt := sel.MatchFirst(tree)
if alt == nil {
return "", errors.New("no feed link found")
}
altUrl, found := FindAttr("href", alt.Attr)
if !found {
return "", errors.New("missing link in alternate")
}
return ToAbsolute(resp.Request.URL, altUrl.Val), nil
}
// Is() checks the current matched set of elements against a selector and
// returns true if at least one of these elements matches.
func (this *Selection) Is(selector string) bool {
if len(this.Nodes) > 0 {
// The selector must be done on the document if it has positional criteria
// TODO : Not sure it is required, as Cascadia's selector checks within the parent of the
// node when there is such a positionaly selector... In jQuery, this is for the
// non-css selectors (Sizzle-implemented selectors, an extension of CSS)
/*if ok, e := regexp.MatchString(rxNeedsContext, selector); ok {
sel := this.document.Root.Find(selector)
for _, n := range this.Nodes {
if sel.IndexOfNode(n) > -1 {
return true
}
}
} else if e != nil {
panic(e.Error())
} else {*/
// Attempt a match with the selector
cs := cascadia.MustCompile(selector)
if len(this.Nodes) == 1 {
return cs.Match(this.Nodes[0])
} else {
return len(cs.Filter(this.Nodes)) > 0
}
//}
}
return false
}
// Is checks the current matched set of elements against a selector and
// returns true if at least one of these elements matches.
func (s *Selection) Is(selector string) bool {
if len(s.Nodes) > 0 {
return s.IsMatcher(cascadia.MustCompile(selector))
}
return false
}
func GetPageInfo(u string) (*PageInfo, error) {
res, err := http.Get(u)
if err != nil {
return nil, err
}
defer res.Body.Close()
tree, err := html.Parse(res.Body)
if err != nil {
return nil, err
}
sel := cascadia.MustCompile("meta")
meta := sel.MatchAll(tree)
found, title := findTitle(tree)
if !found {
_, title = findProperty(meta, "title", "twitter:title")
}
_, description := findProperty(meta, "description", "og:description")
_, image := findProperty(meta, "og:image")
return &PageInfo{
Title: title,
Description: description,
Image: image,
RawURL: u,
}, nil
}
// Is checks the current matched set of elements against a selector and
// returns true if at least one of these elements matches.
func (s *Selection) Is(selector string) bool {
if len(s.Nodes) > 0 {
// Attempt a match with the selector
cs := cascadia.MustCompile(selector)
if len(s.Nodes) == 1 {
return cs.Match(s.Nodes[0])
}
return len(cs.Filter(s.Nodes)) > 0
}
return false
}
// Filter based on a selector string, and the indicator to keep (Filter) or
// to get rid of (Not) the matching elements.
func winnow(sel *Selection, selector string, keep bool) []*html.Node {
cs := cascadia.MustCompile(selector)
// Optimize if keep is requested
if keep {
return cs.Filter(sel.Nodes)
}
// Use grep
return grep(sel, func(i int, s *Selection) bool {
return !cs.Match(s.Get(0))
})
}
// Closest() gets the first element that matches the selector by testing the
// element itself and traversing up through its ancestors in the DOM tree.
func (this *Selection) Closest(selector string) *Selection {
cs := cascadia.MustCompile(selector)
return pushStack(this, mapNodes(this.Nodes, func(i int, n *html.Node) []*html.Node {
// For each node in the selection, test the node itself, then each parent
// until a match is found.
for ; n != nil; n = n.Parent {
if cs.Match(n) {
return []*html.Node{n}
}
}
return nil
}))
}
// Internal implementation of Find that return raw nodes.
func findWithSelector(nodes []*html.Node, selector string) []*html.Node {
// Compile the selector once
sel := cascadia.MustCompile(selector)
// Map nodes to find the matches within the children of each node
return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
// Go down one level, becausejQuery's Find() selects only within descendants
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode {
result = append(result, sel.MatchAll(c)...)
}
}
return
})
}
func findTitle(tree *html.Node) (found bool, title string) {
sel := cascadia.MustCompile("title")
node := sel.MatchFirst(tree)
if node == nil {
return false, ""
}
if node.Type == html.ElementNode {
node = node.FirstChild
}
buf := new(bytes.Buffer)
for node != nil {
if node.Type == html.TextNode {
buf.WriteString(node.Data)
}
node = node.NextSibling
}
return true, string(buf.Bytes())
}
func GetFavicon(url string) (string, error) {
if favicon, err := GetCanonicalFavicon(url); err == nil {
fmt.Println("found favicon.ico")
return favicon, nil
} else if *debug {
fmt.Printf("Error: getting /favicon.ico: %s\n", err)
}
resp, err := http.Get(url)
if *debug {
fmt.Println("get html", resp, err)
}
if err != nil {
return "", err
}
defer resp.Body.Close()
tree, err := html.Parse(resp.Body)
if *debug {
fmt.Println("parse html", tree, err)
}
if err != nil {
return "", err
}
sel := cascadia.MustCompile("link[rel~=icon]")
node := sel.MatchFirst(tree)
if node == nil {
return "", errors.New("no favicon found")
}
favicon, found := FindAttr("href", node.Attr)
if !found {
return "", errors.New("no link found")
}
return ToAbsolute(resp.Request.URL, favicon.Val), nil
}
// Closest gets the first element that matches the selector by testing the
// element itself and traversing up through its ancestors in the DOM tree.
func (s *Selection) Closest(selector string) *Selection {
cs := cascadia.MustCompile(selector)
return s.ClosestMatcher(cs)
}
// WrapInner wraps an HTML structure, matched by the given selector, around the
// content of element in the set of matched elements. The matched child is
// cloned before being inserted into the document.
//
// It returns the original set of elements.
func (s *Selection) WrapInner(selector string) *Selection {
return s.WrapInnerMatcher(cascadia.MustCompile(selector))
}
// Append appends the elements specified by the selector to the end of each element
// in the set of matched elements, following those rules:
//
// 1) The selector is applied to the root document.
//
// 2) Elements that are part of the document will be moved to the new location.
//
// 3) If there are multiple locations to append to, cloned nodes will be
// appended to all target locations except the last one, which will be moved
// as noted in (2).
func (s *Selection) Append(selector string) *Selection {
return s.AppendMatcher(cascadia.MustCompile(selector))
}
// RemoveFiltered removes the set of matched elements by selector.
// It returns the Selection of removed nodes.
func (s *Selection) RemoveFiltered(selector string) *Selection {
return s.RemoveMatcher(cascadia.MustCompile(selector))
}
// ReplaceWith replaces each element in the set of matched elements with the
// nodes matched by the given selector.
// It returns the removed elements.
//
// This follows the same rules as Selection.Append.
func (s *Selection) ReplaceWith(selector string) *Selection {
return s.ReplaceWithMatcher(cascadia.MustCompile(selector))
}
// ParentsUntil gets the ancestors of each element in the Selection, up to but
// not including the element matched by the selector. It returns a new Selection
// object containing the matched elements.
func (s *Selection) ParentsUntil(selector string) *Selection {
return pushStack(s, getParentsNodes(s.Nodes, cascadia.MustCompile(selector), nil))
}
// Before inserts the matched elements before each element in the set of matched elements.
//
// This follows the same rules as Selection.Append.
func (s *Selection) Before(selector string) *Selection {
return s.BeforeMatcher(cascadia.MustCompile(selector))
}
"github.com/knieriem/markdown"
"github.com/redneckbeard/gadget/env"
"github.com/redneckbeard/penny/preprocess"
html "html/template"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
var (
whitespace = regexp.MustCompile(`\W+`)
nonWhitespace = regexp.MustCompile(`[^\w\s]`)
selector = cascadia.MustCompile("p")
)
type Page struct {
*FrontMatter
Body html.HTML
Markdown string
WordCount int
Path string
Next, Previous *Page
Series *series
}
func (p *Page) IsVisible() bool {
switch {
case env.Debug:
// ParentsFilteredUntil is like ParentsUntil, with the option to filter the
// results based on a selector string. It returns a new Selection
// object containing the matched elements.
func (s *Selection) ParentsFilteredUntil(filterSelector, untilSelector string) *Selection {
return filterAndPush(s, getParentsNodes(s.Nodes, cascadia.MustCompile(untilSelector), nil), cascadia.MustCompile(filterSelector))
}
// Not removes elements from the Selection that match the selector string.
// It returns a new Selection object with the matching elements removed.
func (s *Selection) Not(selector string) *Selection {
return s.NotMatcher(cascadia.MustCompile(selector))
}
// PrevUntil gets all preceding siblings of each element up to but not
// including the element matched by the selector. It returns a new Selection
// object containing the matched elements.
func (s *Selection) PrevUntil(selector string) *Selection {
return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
cascadia.MustCompile(selector), nil))
}
// PrevAllFiltered gets all the preceding siblings of each element in the
// Selection filtered by a selector. It returns a new Selection object
// containing the matched elements.
func (s *Selection) PrevAllFiltered(selector string) *Selection {
return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil), cascadia.MustCompile(selector))
}
// ParentsFilteredUntilNodes is like ParentsUntilNodes, with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (s *Selection) ParentsFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
return filterAndPush(s, getParentsNodes(s.Nodes, nil, nodes), cascadia.MustCompile(filterSelector))
}
// Find gets the descendants of each element in the current set of matched
// elements, filtered by a selector. It returns a new Selection object
// containing these matched elements.
func (s *Selection) Find(selector string) *Selection {
return pushStack(s, findWithMatcher(s.Nodes, cascadia.MustCompile(selector)))
}
// PrevFilteredUntilNodes is like PrevUntilNodes, with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (s *Selection) PrevFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
nil, nodes), cascadia.MustCompile(filterSelector))
}
// ChildrenFiltered gets the child elements of each element in the Selection,
// filtered by the specified selector. It returns a new
// Selection object containing these elements.
func (s *Selection) ChildrenFiltered(selector string) *Selection {
return filterAndPush(s, getChildrenNodes(s.Nodes, siblingAll), cascadia.MustCompile(selector))
}
// PrevFilteredUntil is like PrevUntil, with the option to filter
// the results based on a selector string.
// It returns a new Selection object containing the matched elements.
func (s *Selection) PrevFilteredUntil(filterSelector, untilSelector string) *Selection {
return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
cascadia.MustCompile(untilSelector), nil), cascadia.MustCompile(filterSelector))
}
// ParentsFiltered gets the ancestors of each element in the current
// Selection. It returns a new Selection object with the matched elements.
func (s *Selection) ParentsFiltered(selector string) *Selection {
return filterAndPush(s, getParentsNodes(s.Nodes, nil, nil), cascadia.MustCompile(selector))
}
// Filter reduces the set of matched elements to those that match the selector string.
// It returns a new Selection object for this subset of matching elements.
func (s *Selection) Filter(selector string) *Selection {
return s.FilterMatcher(cascadia.MustCompile(selector))
}
// PrevFilteredUntilSelection is like PrevUntilSelection, with the
// option to filter the results based on a selector string. It returns a new
// Selection object containing the matched elements.
func (s *Selection) PrevFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
return s.PrevMatcherUntilSelection(cascadia.MustCompile(filterSelector), sel)
}
