예제 #1
0
파일: section.go 프로젝트: bcampbell/arts
// grab text of matching selector, return as one line
func sectionFromElement(root *html.Node, sel cascadia.Selector) string {
	el := sel.MatchFirst(root)
	if el == nil {
		return ""
	}
	return compressSpace(getTextContent(el))
}
예제 #2
0
파일: util.go 프로젝트: bcampbell/arts
func closest(n *html.Node, sel cascadia.Selector) *html.Node {
	for n != nil {
		if sel.Match(n) {
			break
		}
		n = n.Parent
	}
	return n
}
예제 #3
0
// getElemText returns the innerText of the selected node.
func getElemText(n *html.Node, sel cascadia.Selector) string {
	if node := sel.MatchFirst(n); node != nil {
		return htmlutil.InnerText(node)
	}
	return ""
}
예제 #4
0
파일: tidy.go 프로젝트: bcampbell/arts
// Tidy up extracted content into something that'll produce reasonable html when
// rendered
// - remove comments
// - trim empty text nodes
// - TODO make links absolute
func tidyNode(node *html.Node) {
	var commentSel cascadia.Selector = func(n *html.Node) bool {
		return n.Type == html.CommentNode
	}
	var textSel cascadia.Selector = func(n *html.Node) bool {
		return n.Type == html.TextNode
	}
	var elementSel cascadia.Selector = func(n *html.Node) bool {
		return n.Type == html.ElementNode
	}
	var imgSel cascadia.Selector = func(n *html.Node) bool {
		return n.Type == html.ElementNode && n.DataAtom == atom.Img
	}

	// remove all comments
	for _, n := range commentSel.MatchAll(node) {
		n.Parent.RemoveChild(n)
	}

	leadingSpace := regexp.MustCompile(`^\s+`)
	trailingSpace := regexp.MustCompile(`\s+$`)
	// trim excessive leading/trailing space in text nodes, and cull empty ones
	for _, n := range textSel.MatchAll(node) {
		txt := leadingSpace.ReplaceAllStringFunc(n.Data, func(in string) string {
			if strings.Contains(in, "\n") {
				return "\n"
			} else {
				return " "
			}
		})
		txt = trailingSpace.ReplaceAllStringFunc(n.Data, func(in string) string {
			if strings.Contains(in, "\n") {
				return "\n"
			} else {
				return " "
			}
		})
		if len(strings.TrimSpace(txt)) == 0 {
			n.Parent.RemoveChild(n)
		} else {
			n.Data = txt
		}
	}

	// remove any elements or attrs not on the whitelist
	for _, n := range elementSel.MatchAll(node) {
		allowedAttrs, whiteListed := elementWhitelist[n.DataAtom]
		if !whiteListed {
			if n.Parent != nil {
				n.Parent.RemoveChild(n)
			}
			continue
		}
		filterAttrs(n, func(attr *html.Attribute) bool {
			for _, allowed := range allowedAttrs {
				if attr.Key == allowed.String() {
					return true
				}
			}
			return false
		})
	}

	// special pass for images - strip out ones with huge URIs (eg embedded
	// 'data:' + base64 encoded images)
	const maxSrcURI = 1024
	for _, img := range imgSel.MatchAll(node) {
		src := getAttr(img, "src")
		if len(src) > maxSrcURI {
			img.Parent.RemoveChild(img)
			continue
		}
	}
}