Пример #1
0
func setClasses(n *html.Node, attr *html.Attribute, classes string) {
	classes = strings.TrimSpace(classes)
	if classes == "" {
		removeAttr(n, "class")
		return
	}

	attr.Val = classes
}
Пример #2
0
func (p Parser) setAttr(selection *goquery.Selection, attr string, value string) {
	if selection.Size() > 0 {
		node := selection.Get(0)
		var attrs []html.Attribute
		for _, a := range node.Attr {
			if a.Key != attr {
				newAttr := new(html.Attribute)
				newAttr.Key = a.Key
				newAttr.Val = a.Val
				attrs = append(attrs, *newAttr)
			}
		}
		newAttr := new(html.Attribute)
		newAttr.Key = attr
		newAttr.Val = value
		attrs = append(attrs, *newAttr)
		node.Attr = attrs
	}
}
Пример #3
0
func cleanURL(c *Config, a atom.Atom, attr *html.Attribute) bool {
	if a != atom.Href && a != atom.Src && a != atom.Poster {
		return true
	}

	u, err := url.Parse(attr.Val)
	if err != nil {
		return false
	}
	if c.ValidateURL != nil && !c.ValidateURL(u) {
		return false
	}
	attr.Val = u.String()
	return true
}
Пример #4
0
func nodesCleanup(nodes []*html.Node) bool {
	changed := false

	for _, n := range nodes {
		if n.Type == html.ElementNode && n.Data == "script" || n.Type == html.CommentNode {
			if n.Parent != nil {
				n.Parent.RemoveChild(n)
				changed = true
			}
			break
		}

		if n.Type == html.ElementNode {
			// Remove all 'on*' attributes, and any that contain 'javascript:'
			attrs := []html.Attribute{}
			for _, a := range n.Attr {
				if strings.HasPrefix(a.Key, "on") {
					changed = true
					break
				}

				i := strings.Index(a.Val, "javascript:")
				if i != -1 {
					onlySpace := true
					if i > 0 {
						for _, r := range a.Val[:i] {
							if !unicode.IsSpace(r) {
								onlySpace = false
								break
							}
						}
					}

					if onlySpace {
						changed = true
						break
					}
				}

				attrs = append(attrs, a)
			}

			n.Attr = attrs

			// Add a target attribute to the article links
			if n.Data == "a" {
				var attr *html.Attribute

				for i, a := range n.Attr {
					if a.Key == "target" {
						attr = &n.Attr[i]
						break
					}
				}

				val := "feed-article"
				if attr == nil {
					n.Attr = append(n.Attr, html.Attribute{Key: "target", Val: val})
				} else {
					attr.Val = val
				}
				changed = true
			}
		}

		children := []*html.Node{}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			children = append(children, c)
		}

		if len(children) > 0 {
			if nodesCleanup(children) {
				changed = true
			}
		}
	}

	return changed
}
Пример #5
0
// sanitizeAttrs takes a set of element attribute policies and the global
// attribute policies and applies them to the []html.Attribute returning a set
// of html.Attributes that match the policies
func (p *Policy) sanitizeAttrs(
	elementName string,
	attrs []html.Attribute,
	aps map[string]attrPolicy,
) []html.Attribute {

	if len(attrs) == 0 {
		return attrs
	}

	// Builds a new attribute slice based on the whether the attribute has been
	// whitelisted explicitly or globally.
	cleanAttrs := []html.Attribute{}
	for _, htmlAttr := range attrs {
		// Is there an element specific attribute policy that applies?
		if ap, ok := aps[htmlAttr.Key]; ok {
			if ap.regexp != nil {
				if ap.regexp.MatchString(htmlAttr.Val) {
					cleanAttrs = append(cleanAttrs, htmlAttr)
					continue
				}
			} else {
				cleanAttrs = append(cleanAttrs, htmlAttr)
				continue
			}
		}

		// Is there a global attribute policy that applies?
		if ap, ok := p.globalAttrs[htmlAttr.Key]; ok {
			if ap.regexp != nil {
				if ap.regexp.MatchString(htmlAttr.Val) {
					cleanAttrs = append(cleanAttrs, htmlAttr)
				}
			} else {
				cleanAttrs = append(cleanAttrs, htmlAttr)
			}
		}
	}

	if len(cleanAttrs) == 0 {
		// If nothing was allowed, let's get out of here
		return cleanAttrs
	}
	// cleanAttrs now contains the attributes that are permitted

	if linkable(elementName) {
		if p.requireParseableURLs {
			// Ensure URLs are parseable:
			// - a.href
			// - area.href
			// - link.href
			// - blockquote.cite
			// - q.cite
			// - img.src
			// - script.src
			tmpAttrs := []html.Attribute{}
			for _, htmlAttr := range cleanAttrs {
				switch elementName {
				case "a", "area", "link":
					if htmlAttr.Key == "href" {
						if u, ok := p.validURL(htmlAttr.Val); ok {
							htmlAttr.Val = u
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
						break
					}
					tmpAttrs = append(tmpAttrs, htmlAttr)
				case "blockquote", "q":
					if htmlAttr.Key == "cite" {
						if u, ok := p.validURL(htmlAttr.Val); ok {
							htmlAttr.Val = u
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
						break
					}
					tmpAttrs = append(tmpAttrs, htmlAttr)
				case "img", "script":
					if htmlAttr.Key == "src" {
						if u, ok := p.validURL(htmlAttr.Val); ok {
							htmlAttr.Val = u
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
						break
					}
					tmpAttrs = append(tmpAttrs, htmlAttr)
				default:
					tmpAttrs = append(tmpAttrs, htmlAttr)
				}
			}
			cleanAttrs = tmpAttrs
		}

		if (p.requireNoFollow ||
			p.requireNoFollowFullyQualifiedLinks ||
			p.addTargetBlankToFullyQualifiedLinks) &&
			len(cleanAttrs) > 0 {

			// Add rel="nofollow" if a "href" exists
			switch elementName {
			case "a", "area", "link":
				var hrefFound bool
				var externalLink bool
				for _, htmlAttr := range cleanAttrs {
					if htmlAttr.Key == "href" {
						hrefFound = true

						u, err := url.Parse(htmlAttr.Val)
						if err != nil {
							continue
						}
						if u.Host != "" {
							externalLink = true
						}

						continue
					}
				}

				if hrefFound {
					var noFollowFound bool
					var targetFound bool

					addNoFollow := (p.requireNoFollow ||
						externalLink && p.requireNoFollowFullyQualifiedLinks)

					addTargetBlank := (externalLink &&
						p.addTargetBlankToFullyQualifiedLinks)

					tmpAttrs := []html.Attribute{}
					for _, htmlAttr := range cleanAttrs {

						var appended bool
						if htmlAttr.Key == "rel" && addNoFollow {

							if strings.Contains(htmlAttr.Val, "nofollow") {
								noFollowFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
							} else {
								htmlAttr.Val += " nofollow"
								noFollowFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
							}

							appended = true
						}

						if elementName == "a" &&
							htmlAttr.Key == "target" &&
							addTargetBlank {

							if strings.Contains(htmlAttr.Val, "_blank") {
								targetFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
							} else {
								htmlAttr.Val = "_blank"
								targetFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
							}

							appended = true
						}

						if !appended {
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
					}
					if noFollowFound || targetFound {
						cleanAttrs = tmpAttrs
					}

					if addNoFollow && !noFollowFound {
						rel := html.Attribute{}
						rel.Key = "rel"
						rel.Val = "nofollow"
						cleanAttrs = append(cleanAttrs, rel)
					}

					if elementName == "a" && addTargetBlank && !targetFound {
						rel := html.Attribute{}
						rel.Key = "target"
						rel.Val = "_blank"
						cleanAttrs = append(cleanAttrs, rel)
					}
				}
			default:
			}
		}
	}

	return cleanAttrs
}
Пример #6
0
//从nodes中找到node  根据index  和 属性  先index
func findNodeformNodesbyIndexOrPro(nodes []*goquery.Selection, index *int, m map[string]string, Type string, visible bool) {

	switch {
	case Type == OPTION || Type == RADIO:

		for _, v := range nodes {
			for _, vv := range v.Get(0).Attr {
				if vv.Key == VALUE {

					if vv.Val == m[VALUE] {
						if Type == RADIO {
							v.SetAttr("checked", "checked")
						} else {
							v.SetAttr("selected", "selected")
						}

						return
					}

				}
			}
		}
		if visible {
			var node html.Node
			node.Data = nodes[0].Get(0).Data
			node.Type = nodes[0].Get(0).Type

			attr := make([]html.Attribute, 0, 2)
			var tr html.Attribute
			tr.Key = VALUE
			tr.Val = m[VALUE]

			attr = append(attr, tr)
			if Type == RADIO {
				tr.Key = "checked"
				tr.Val = "checked"
			} else {
				tr.Key = "selected"
				tr.Val = "selected"
			}

			attr = append(attr, tr)

			tr.Key = TYPE
			tr.Val = Type
			attr = append(attr, tr)

			node.Attr = attr
			nodes[0].Parent().AppendNodes(&node)
		}
		return
	default:
	}

	if len(nodes) <= *index {
		return
	}
	for k, v := range m {
		nodes[*index].SetAttr(k, v)
	}

	*index++
}
Пример #7
0
// sanitizeAttrs takes a set of element attribute policies and the global
// attribute policies and applies them to the []html.Attribute returning a set
// of html.Attributes that match the policies
func (p *Policy) sanitizeAttrs(
	elementName string,
	attrs []html.Attribute,
	aps map[string]attrPolicy,
) []html.Attribute {

	if len(attrs) == 0 {
		return attrs
	}

	// Builds a new attribute slice based on the whether the attribute has been
	// whitelisted explicitly or globally.
	cleanAttrs := []html.Attribute{}
	for _, htmlAttr := range attrs {
		// Is there an element specific attribute policy that applies?
		if ap, ok := aps[htmlAttr.Key]; ok {
			if ap.regexp != nil {
				if ap.regexp.MatchString(htmlAttr.Val) {
					cleanAttrs = append(cleanAttrs, htmlAttr)
					continue
				}
			} else {
				cleanAttrs = append(cleanAttrs, htmlAttr)
				continue
			}
		}

		// Is there a global attribute policy that applies?
		if ap, ok := p.globalAttrs[htmlAttr.Key]; ok {
			if ap.regexp != nil {
				if ap.regexp.MatchString(htmlAttr.Val) {
					cleanAttrs = append(cleanAttrs, htmlAttr)
				}
			} else {
				cleanAttrs = append(cleanAttrs, htmlAttr)
			}
		}
	}

	if len(cleanAttrs) == 0 {
		// If nothing was allowed, let's get out of here
		return cleanAttrs
	}
	// cleanAttrs now contains the attributes that are permitted

	if linkable(elementName) {
		if p.requireParseableURLs {
			// Ensure URLs are parseable:
			// - a.href
			// - area.href
			// - link.href
			// - blockquote.cite
			// - q.cite
			// - img.src
			// - script.src
			tmpAttrs := []html.Attribute{}
			for _, htmlAttr := range cleanAttrs {
				switch elementName {
				case "a", "area", "link":
					if htmlAttr.Key == "href" {
						if u, ok := p.validURL(htmlAttr.Val); ok {
							htmlAttr.Val = u
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
						break
					}
					tmpAttrs = append(tmpAttrs, htmlAttr)
				case "blockquote", "q":
					if htmlAttr.Key == "cite" {
						if u, ok := p.validURL(htmlAttr.Val); ok {
							htmlAttr.Val = u
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
						break
					}
					tmpAttrs = append(tmpAttrs, htmlAttr)
				case "img", "script":
					if htmlAttr.Key == "src" {
						if u, ok := p.validURL(htmlAttr.Val); ok {
							htmlAttr.Val = u
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
						break
					}
					tmpAttrs = append(tmpAttrs, htmlAttr)
				default:
					tmpAttrs = append(tmpAttrs, htmlAttr)
				}
			}
			cleanAttrs = tmpAttrs
		}

		if (p.requireNoFollow ||
			p.requireNoFollowFullyQualifiedLinks ||
			p.addTargetBlankToFullyQualifiedLinks) &&
			len(cleanAttrs) > 0 {

			// Add rel="nofollow" if a "href" exists
			switch elementName {
			case "a", "area", "link":
				var hrefFound bool
				var externalLink bool
				for _, htmlAttr := range cleanAttrs {
					if htmlAttr.Key == "href" {
						hrefFound = true

						u, err := url.Parse(htmlAttr.Val)
						if err != nil {
							continue
						}
						if u.Host != "" {
							externalLink = true
						}

						continue
					}
				}

				if hrefFound {
					var (
						noFollowFound    bool
						targetBlankFound bool
					)

					addNoFollow := (p.requireNoFollow ||
						externalLink && p.requireNoFollowFullyQualifiedLinks)

					addTargetBlank := (externalLink &&
						p.addTargetBlankToFullyQualifiedLinks)

					tmpAttrs := []html.Attribute{}
					for _, htmlAttr := range cleanAttrs {

						var appended bool
						if htmlAttr.Key == "rel" && addNoFollow {

							if strings.Contains(htmlAttr.Val, "nofollow") {
								noFollowFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
								appended = true
							} else {
								htmlAttr.Val += " nofollow"
								noFollowFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
								appended = true
							}
						}

						if elementName == "a" && htmlAttr.Key == "target" {
							if htmlAttr.Val == "_blank" {
								targetBlankFound = true
							}
							if addTargetBlank && !targetBlankFound {
								htmlAttr.Val = "_blank"
								targetBlankFound = true
								tmpAttrs = append(tmpAttrs, htmlAttr)
								appended = true
							}
						}

						if !appended {
							tmpAttrs = append(tmpAttrs, htmlAttr)
						}
					}
					if noFollowFound || targetBlankFound {
						cleanAttrs = tmpAttrs
					}

					if addNoFollow && !noFollowFound {
						rel := html.Attribute{}
						rel.Key = "rel"
						rel.Val = "nofollow"
						cleanAttrs = append(cleanAttrs, rel)
					}

					if elementName == "a" && addTargetBlank && !targetBlankFound {
						rel := html.Attribute{}
						rel.Key = "target"
						rel.Val = "_blank"
						targetBlankFound = true
						cleanAttrs = append(cleanAttrs, rel)
					}

					if targetBlankFound {
						// target="_blank" has a security risk that allows the
						// opened window/tab to issue JavaScript calls against
						// window.opener, which in effect allow the destination
						// of the link to control the source:
						// https://dev.to/ben/the-targetblank-vulnerability-by-example
						//
						// To mitigate this risk, we need to add a specific rel
						// attribute if it is not already present.
						// rel="noopener"
						//
						// Unfortunately this is processing the rel twice (we
						// already looked at it earlier ^^) as we cannot be sure
						// of the ordering of the href and rel, and whether we
						// have fully satisfied that we need to do this. This
						// double processing only happens *if* target="_blank"
						// is true.
						var noOpenerAdded bool
						tmpAttrs := []html.Attribute{}
						for _, htmlAttr := range cleanAttrs {
							var appended bool
							if htmlAttr.Key == "rel" {
								if strings.Contains(htmlAttr.Val, "noopener") {
									noOpenerAdded = true
									tmpAttrs = append(tmpAttrs, htmlAttr)
								} else {
									htmlAttr.Val += " noopener"
									noOpenerAdded = true
									tmpAttrs = append(tmpAttrs, htmlAttr)
								}

								appended = true
							}
							if !appended {
								tmpAttrs = append(tmpAttrs, htmlAttr)
							}
						}
						if noOpenerAdded {
							cleanAttrs = tmpAttrs
						} else {
							// rel attr was not found, or else noopener would
							// have been added already
							rel := html.Attribute{}
							rel.Key = "rel"
							rel.Val = "noopener"
							cleanAttrs = append(cleanAttrs, rel)
						}

					}
				}
			default:
			}
		}
	}

	return cleanAttrs
}