func (b *Bot) SendHtml(chat xmpp.Chat) {
text := strings.Replace(chat.Text, "&", "&", -1)
org := fmt.Sprintf("<message to='%s' type='%s' xml:lang='en'><body>%s</body>"+
"<html xmlns='http://jabber.org/protocol/xhtml-im'><body xmlns='http://www.w3.org/1999/xhtml'>%s</body></html></message>",
html.EscapeString(chat.Remote), html.EscapeString(chat.Type), html.EscapeString(chat.Text), text)
b.client.SendOrg(org)
}
// Preprocess escapes disallowed tags in a cleaner way, but does not fix
// nesting problems. Use with Clean.
func Preprocess(config *Config, fragment string) string {
if config == nil {
config = DefaultConfig
}
var buf bytes.Buffer
write := func(raw string) {
_, err := buf.WriteString(raw)
// The only possible error is running out of memory.
expectError(err, nil)
}
t := html.NewTokenizer(strings.NewReader(fragment))
for {
switch tok := t.Next(); tok {
case html.ErrorToken:
err := t.Err()
// The only possible errors are from the Reader or from
// the buffer capacity being exceeded. Neither can
// happen with strings.NewReader as the string must
// already fit into memory.
expectError(err, io.EOF)
if err == io.EOF {
write(html.EscapeString(string(t.Raw())))
return buf.String()
}
case html.TextToken:
write(string(t.Raw()))
case html.StartTagToken, html.EndTagToken, html.SelfClosingTagToken:
raw := string(t.Raw())
tagName, _ := t.TagName()
allowed := false
if tag := atom.Lookup(tagName); tag != 0 {
if _, ok := config.elem[tag]; ok {
allowed = true
}
}
if !allowed {
if _, ok := config.elemCustom[string(tagName)]; ok {
allowed = true
}
}
if !allowed {
raw = html.EscapeString(raw)
}
write(raw)
case html.CommentToken:
raw := string(t.Raw())
if config.EscapeComments || !strings.HasPrefix(raw, "<!--") || !strings.HasSuffix(raw, "-->") {
raw = html.EscapeString(raw)
}
write(raw)
default:
write(html.EscapeString(string(t.Raw())))
}
}
}
// FixHtml parses bytes as HTML and returns well-formed HTML if the parse
// was successful, or escaped HTML, if not.
func fixHtml(linkUrl string, wild []byte) (well []byte) {
n, err := html.Parse(bytes.NewReader(wild))
if err != nil {
return []byte(html.EscapeString(string(wild)))
}
fixImgs(linkUrl, n)
defer func() {
if err := recover(); err == bytes.ErrTooLarge {
well = []byte(html.EscapeString(string(wild)))
} else if err != nil {
panic(err)
}
}()
buf := bytes.NewBuffer(make([]byte, 0, len(wild)*2))
if err := html.Render(buf, n); err != nil {
return []byte(html.EscapeString(string(wild)))
}
well = buf.Bytes()
openBody := []byte("<body>")
i := bytes.Index(well, openBody)
if i < 0 {
return []byte(html.EscapeString(string(wild)))
}
well = well[i+len(openBody):]
closeBody := []byte("</body>")
i = bytes.Index(well, closeBody)
if i < 0 {
return []byte(html.EscapeString(string(wild)))
}
return well[:i]
}
func (c *KvController) Get(rw http.ResponseWriter, r *http.Request) error {
record := c.Repo.Get(html.EscapeString(r.URL.Path))
if len(record) == 0 {
rw.WriteHeader(http.StatusNotFound)
return nil
}
rw.Write([]byte(record))
c.EventBus.Publish(EVENT_VALUE_RETRIEVED, KeyValuePair{Key: html.EscapeString(r.URL.Path), Value: record})
return nil
}
// render renders a simplified version of the provided HTML node. It will strip
// doctype nodes, trim spaces on text nodes and insert indentation.
//
// Note: render doesn't guarantee that the semantics of the page are preserved.
func render(w io.Writer, n *html.Node, indent int) {
switch n.Type {
case html.ElementNode:
fmt.Fprintf(w, strings.Repeat(" ", indent))
fmt.Fprintf(w, "<%s", n.Data)
for _, attr := range n.Attr {
fmt.Fprintf(w, ` %s="%s"`, attr.Key, html.EscapeString(attr.Val))
}
fmt.Fprintf(w, ">\n")
case html.TextNode:
renderText(w, n.Data, indent)
case html.CommentNode:
if len(strings.TrimSpace(n.Data)) == 0 {
// skip empty comments.
break
}
fmt.Fprintf(w, "<!--\n")
renderText(w, n.Data, indent)
fmt.Fprintf(w, "-->\n")
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
i := indent + 1
if n.Type == html.DocumentNode {
// don't increase indentation after void elements.
i = indent
}
render(w, c, i)
}
if n.Type == html.ElementNode && !voidElements[n.Data] {
// close non-void elements
fmt.Fprintf(w, strings.Repeat(" ", indent))
fmt.Fprintf(w, "</%s>\n", n.Data)
}
}
func (c *KvController) Put(rw http.ResponseWriter, r *http.Request) error {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
err = c.Repo.Put(html.EscapeString(r.URL.Path), string(body))
if err != nil {
return err
}
rw.WriteHeader(http.StatusCreated)
c.EventBus.Publish(EVENT_VALUE_STORED, KeyValuePair{Key: html.EscapeString(r.URL.Path), Value: string(body)})
return nil
}
func (a *Article) addInlineArticleImageHTML(title string) {
if a.Img == nil {
return
}
if a.TopNode == nil {
a.TopNode = goquery.NewDocumentFromNode(&html.Node{
Type: html.ElementNode,
DataAtom: atom.Span,
Data: "span",
}).Selection
}
a.TopNode.PrependHtml(fmt.Sprintf(imgHeader,
html.EscapeString(a.URL),
html.EscapeString(title),
html.EscapeString(a.Img.Src)))
}
func editLinkTags(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.NotFound(w, r)
return
}
c := appengine.NewContext(r)
u := user.Current(c)
lu, uk, err := getUser(c)
if err != nil {
showError(w, "failed to retrieve user", http.StatusInternalServerError, c)
c.Errorf("failed to retrieve user %q: %v", u.String(), err)
return
}
var k *datastore.Key
if k, err = datastore.DecodeKey(r.FormValue("Key")); err != nil {
http.Error(w, err.Error(), 501)
return
}
l := new(Link)
if err := datastore.Get(c, k, l); err != nil {
http.Error(w, err.Error(), 500)
return
}
tags := html.EscapeString(r.FormValue("Tags"))
l.Tags = []string{}
tagList := strings.Split(tags, ",")
for _, tag := range tagList {
tag = strings.TrimSpace(tag)
if tag != "" {
l.Tags = append(l.Tags, tag)
}
}
nt := len(lu.Tags)
lu.AddTags(l.Tags)
if len(lu.Tags) != nt {
err := lu.Save(c, uk)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
if _, err := datastore.Put(c, k, l); err != nil {
http.Error(w, err.Error(), 500)
return
}
}
func renderText(w io.Writer, data string, indent int) {
lines := strings.Split(html.EscapeString(data), "\n")
for _, line := range lines {
s := strings.TrimSpace(line)
if len(s) == 0 {
continue
}
fmt.Fprintf(w, strings.Repeat(" ", indent))
fmt.Fprintf(w, "%s\n", s)
}
}
func joinAttrs(attrs []html.Attribute) string {
buff := bytes.NewBufferString("")
for i, a := range attrs {
if i > 0 {
buff.WriteString(" ")
}
if a.Namespace != "" {
buff.WriteString(a.Namespace)
buff.WriteString(":")
}
buff.WriteString(a.Key)
buff.WriteString(`="`)
buff.WriteString(html.EscapeString(a.Val))
buff.WriteString(`"`)
}
return buff.String()
}
func editLinkTitle(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.NotFound(w, r)
return
}
c := appengine.NewContext(r)
u := user.Current(c)
_, _, err := getUser(c)
if err != nil {
showError(w, "failed to retrieve user", http.StatusInternalServerError, c)
c.Errorf("failed to retrieve user %q: %v", u.String(), err)
return
}
var k *datastore.Key
if k, err = datastore.DecodeKey(r.FormValue("Key")); err != nil {
http.Error(w, err.Error(), 501)
return
}
l := new(Link)
if err := datastore.Get(c, k, l); err != nil {
http.Error(w, err.Error(), 500)
return
}
l.Title = html.EscapeString(r.FormValue("Title"))
if _, err := datastore.Put(c, k, l); err != nil {
http.Error(w, err.Error(), 500)
return
}
}
// TestRemoveTags ...
func TestRemoveTags(t *testing.T) {
tags := []string{
"<script>i=0;</script>",
"<button>text</button>",
"<object>text</object>",
"<select>text</select>",
"<style>.a: 1px</style>",
"<param name=\"a\"/>",
"<embed a=\"1\"></embed>",
"<form>text</form>",
"<img src=\"URL\"></img>",
"<svg>text</svg>",
"<canvas>text</canvas>",
"<video>text</video>",
"<textarea>text</textarea>",
"<noscript>text</noscript>",
"<applet>text</applet>",
"<audio>text</audio>",
"<noindex>text<div>text</div>text</noindex>",
`<iframe src="page/banner.html" width="468" height="60" align="left">
Ваш браузер не поддерживает встроенные фреймы!</iframe>`,
`<input type="hidden"/>`,
`<command onclick="alert">`,
"<area/>",
"<br/>",
"<hr/>",
}
for _, tagName := range tags {
helperBody("Removing tag "+html.EscapeString(tagName), t,
fmt.Sprintf("pre%spost", tagName),
"pre post")
}
tags = []string{
"<title>text</title>",
`<meta name="title" content="title meta">`,
`<link rel="stylesheet" href="ie.css">`,
`<base href="ie.css">`,
`<basefont face="Arial">`,
`<bgsound src="1.ogg">`,
}
for _, tagName := range tags {
helperHead("Removing tag "+html.EscapeString(tagName), t, tagName, "")
}
tags = []string{
`<data value="1"></data>`,
`<datalist id="<идентификатор>"><option value="text1"></datalist>`,
}
for _, tagName := range tags {
helperDiv("Removing tag "+html.EscapeString(tagName), t,
fmt.Sprintf("pre%spost", tagName),
"prepost")
}
helperBody("Removing tag wdr", t,
"pre<wbr>post",
"prepost")
helperDiv("Removing tag wdr between tags", t,
"pre</div><wbr><div>post",
"pre</div><div>post")
Convey("Removing tag frameset", t, func() {
in := `<html><head></head><frameset></frameset></html>`
out := "<html><head></head></html>"
minificationCheck(in, out)
})
}
/*
render for a start tag containing commands.
If a tal:omit-tag TALES expression is present, it is evaluated.
If the element is not a void element (e.g. <img>), the resulting
true or false is stored in the render context for the end tag to pickup.
If a contentExpression (from tal:content or tal:replace) is present, it is
evaluated. If it resolves to Default or it is a tal:content command and
tal:omit-tag is missing or is false, the start tag is rendered.
Start tag rendering checks whether there are any attribute expressions. If
there are, these are evaluated and the effective attributes updated.
If there is a tal:replace command that did not evaluate to Default, execution
jumps to the end tag.
*/
func (d *renderStartTag) render(rc *renderContext) error {
// If tal:omit-tag has been used, always ensure that we have called addOmitTagFlag()
omitTagFlag := false
if d.omitTagExpression != "" {
omitTagValue := rc.talesContext.evaluate(d.omitTagExpression, d.originalAttributes)
omitTagFlag = trueOrFalse(omitTagValue)
// Add this onto the context
rc.debug("Omit Tag Flag %v - Omit Tag Value %v - Void %v\n", omitTagFlag, omitTagValue, d.voidElement)
if !d.voidElement {
rc.addOmitTagFlag(omitTagFlag)
}
}
var contentValue interface{}
if d.contentExpression != "" {
contentValue = rc.talesContext.evaluate(d.contentExpression, d.originalAttributes)
}
rc.debug("Start tag content is %v\n", contentValue)
rc.buffer.reset()
if contentValue == Default || (!d.replaceCommand && !omitTagFlag) {
// We are going to write out a start tag, so it's worth evaluating any tal:attribute values at this point.
var attributes attributesList
if len(d.attributeExpression) == 0 {
// No tal:attributes - just use the original values.
attributes = d.originalAttributes
} else {
// Start by taking a copy of the original attributes
attributes = append(attributes, d.originalAttributes...)
// Now evaluate each tal:attribute and see what needs to be done.
for _, talAtt := range d.attributeExpression {
attValue := rc.talesContext.evaluate(talAtt.Val, d.originalAttributes)
if attValue == nil {
// Need to remove this attribute from the list.
attributes.Remove(talAtt.Key)
} else if attValue != Default {
// Over-ride the value
// If it's a boolean attribute, use the expression to determine what to do.
_, booleanAtt := htmlBooleanAttributes[talAtt.Key]
if booleanAtt {
if trueOrFalse(attValue) {
// True boolean attributes get the value of their name
attributes.Set(talAtt.Key, talAtt.Key)
} else {
// We remove the attribute
attributes.Remove(talAtt.Key)
}
} else {
// Normal attribute - just set to the string value.
attributes.Set(talAtt.Key, fmt.Sprint(attValue))
}
}
}
}
rc.buffer.appendString("<")
rc.buffer.append(d.tagName)
for _, att := range attributes {
rc.buffer.appendString(" ")
rc.buffer.appendString(att.Key)
rc.buffer.appendString("=\"")
rc.buffer.appendString(html.EscapeString(att.Val))
rc.buffer.appendString("\"")
}
rc.buffer.appendString(">")
rc.out.Write(rc.buffer)
}
if contentValue == Default || d.contentExpression == "" {
return nil
}
if contentValue != nil {
if d.contentStructure {
rc.out.Write([]byte(fmt.Sprint(contentValue)))
} else {
rc.out.Write([]byte(html.EscapeString(fmt.Sprint(contentValue))))
}
}
if d.replaceCommand && !d.voidElement {
rc.debug("Omit Tag is true, jumping to +%v\n", d.endTagOffset)
rc.instructionPointer += d.endTagOffset
} else {
rc.instructionPointer += d.endTagOffset - 1
}
return nil
}
// Sanitizes the given input by parsing it as HTML5, then whitelisting known to
// be safe elements and attributes. All other HTML is escaped, unsafe attributes
// are stripped.
func sanitizeHtmlSafe(input []byte) []byte {
r := bytes.NewReader(input)
var w bytes.Buffer
tokenizer := html.NewTokenizer(r)
wr := bufio.NewWriter(&w)
// Iterate through all tokens in the input stream and sanitize them.
for t := tokenizer.Next(); t != html.ErrorToken; t = tokenizer.Next() {
switch t {
case html.TextToken:
// Text is written escaped.
wr.WriteString(tokenizer.Token().String())
case html.SelfClosingTagToken, html.StartTagToken:
// HTML tags are escaped unless whitelisted.
tag, hasAttributes := tokenizer.TagName()
tagName := string(tag)
if whitelistTags[tagName] {
wr.WriteString("<")
wr.Write(tag)
for hasAttributes {
var key, val []byte
key, val, hasAttributes = tokenizer.TagAttr()
attrName := string(key)
// Only include whitelisted attributes for the given tagName.
tagWhitelistedAttrs, ok := whitelistAttrs[tagName]
if ok && tagWhitelistedAttrs[attrName] {
// For whitelisted attributes, if it's an attribute that requires
// protocol checking, do so and strip it if it's not known to be safe.
tagProtocolAttrs, ok := protocolAttrs[tagName]
if ok && tagProtocolAttrs[attrName] {
if !isRelativeLink(val) && !protocolAllowed(val) {
continue
}
}
wr.WriteByte(' ')
wr.Write(key)
wr.WriteString(`="`)
wr.WriteString(html.EscapeString(string(val)))
wr.WriteByte('"')
}
}
if t == html.SelfClosingTagToken {
wr.WriteString("/>")
} else {
wr.WriteString(">")
}
} else {
wr.WriteString(html.EscapeString(string(tokenizer.Raw())))
}
// Make sure that tags like <script> that switch the parser into raw mode
// do not destroy the parse mode for following HTML text (the point is to
// escape them anyway). For that, switch off raw mode in the tokenizer.
tokenizer.NextIsNotRawText()
case html.EndTagToken:
// Whitelisted tokens can be written in raw.
tag, _ := tokenizer.TagName()
if whitelistTags[string(tag)] {
wr.Write(tokenizer.Raw())
} else {
wr.WriteString(html.EscapeString(string(tokenizer.Raw())))
}
case html.CommentToken:
// Comments are not really expected, but harmless.
wr.Write(tokenizer.Raw())
case html.DoctypeToken:
// Escape DOCTYPES, entities etc can be dangerous
wr.WriteString(html.EscapeString(string(tokenizer.Raw())))
default:
tokenizer.Token()
panic(fmt.Errorf("Unexpected token type %v", t))
}
}
err := tokenizer.Err()
if err != nil && err != io.EOF {
panic(tokenizer.Err())
}
wr.Flush()
return w.Bytes()
}
// Minify returns minified version of the given HTML data.
// If passed options is nil, uses default options.
func MinifyHTML(data []byte, options *Options) (out []byte, err error) {
if options == nil {
options = DefaultOptions
}
var b bytes.Buffer
z := html.NewTokenizer(bytes.NewReader(data))
raw := false
javascript := false
style := false
for {
tt := z.Next()
switch tt {
case html.ErrorToken:
err := z.Err()
if err == io.EOF {
return b.Bytes(), nil
}
return nil, err
case html.StartTagToken, html.SelfClosingTagToken:
tagName, hasAttr := z.TagName()
switch string(tagName) {
case "script":
javascript = true
raw = true
case "style":
style = true
raw = true
case "pre", "code", "textarea":
raw = true
default:
raw = false
}
b.WriteByte('<')
b.Write(tagName)
var k, v []byte
isFirst := true
for hasAttr {
k, v, hasAttr = z.TagAttr()
if javascript && string(k) == "type" && string(v) != "text/javascript" {
javascript = false
}
if string(k) == "style" && options.MinifyStyles {
v = []byte("a{" + string(v) + "}") // simulate "full" CSS
v = MinifyCSS(v)
v = v[2 : len(v)-1] // strip simulation
}
if isFirst {
b.WriteByte(' ')
isFirst = false
}
b.Write(k)
b.WriteByte('=')
if quoteChar := valueQuoteChar(v); quoteChar != 0 {
// Quoted value.
b.WriteByte(quoteChar)
b.WriteString(html.EscapeString(string(v)))
b.WriteByte(quoteChar)
} else {
// Unquoted value.
b.Write(v)
}
if hasAttr {
b.WriteByte(' ')
}
}
b.WriteByte('>')
case html.EndTagToken:
tagName, _ := z.TagName()
raw = false
if javascript && string(tagName) == "script" {
javascript = false
}
if style && string(tagName) == "style" {
style = false
}
b.Write([]byte("</"))
b.Write(tagName)
b.WriteByte('>')
case html.CommentToken:
if bytes.HasPrefix(z.Raw(), []byte("<!--[if")) ||
bytes.HasPrefix(z.Raw(), []byte("<!--//")) {
// Preserve IE conditional and special style comments.
b.Write(z.Raw())
}
// ... otherwise, skip.
case html.TextToken:
if javascript && options.MinifyScripts {
min, err := MinifyJS(z.Raw())
if err != nil {
// Just write it as is.
b.Write(z.Raw())
} else {
b.Write(min)
}
} else if style && options.MinifyStyles {
b.Write(MinifyCSS(z.Raw()))
} else if raw {
b.Write(z.Raw())
} else {
b.Write(trimTextToken(z.Raw()))
}
default:
b.Write(z.Raw())
}
}
}
/*
CompileTemplate reads the template in and compiles it ready for execution.
If a compilation error (rather than IO error) occurs, the returned error
will be a CompileError object.
The io.Reader must provide a stream of UTF-8 encoded text. Templates
render into UTF-8, so any conversion to or from other character sets must be
carried out in the io.Reader and io.Writer used.
*/
func CompileTemplate(in io.Reader) (template *Template, err error) {
tokenizer := html.NewTokenizer(in)
template = newTemplate()
state := &compileState{template: template, tokenizer: tokenizer}
for {
token := tokenizer.Next()
switch token {
case html.ErrorToken:
if tokenizer.Err() == io.EOF {
return template, nil
}
return nil, tokenizer.Err()
case html.TextToken:
var d buffer
// Text() returns a []byte that may change, so we immediately make a copy
d.appendString(html.EscapeString(string(tokenizer.Text())))
template.addRenderInstruction(d)
case html.StartTagToken:
rawTagName, hasAttr := tokenizer.TagName()
// rawTagName is a slice of bytes that may change when next() is called on the tokenizer.
// To avoid subtle bugs we create a copy of the data that we know will be immutable
tagName := make([]byte, len(rawTagName))
copy(tagName, rawTagName)
// Note the tag
var voidElement bool = htmlVoidElements[string(tagName)]
state.addTag(tagName)
var d buffer
var originalAtts []html.Attribute
var talAtts []html.Attribute
var key, val, rawkey, rawval []byte
for hasAttr {
rawkey, rawval, hasAttr = tokenizer.TagAttr()
// TagAttr returns slides that may change when next() is called - so duplicate them before casting to string
key = make([]byte, len(rawkey))
copy(key, rawkey)
val = make([]byte, len(rawval))
copy(val, rawval)
att := html.Attribute{Key: string(key), Val: string(val)}
if strings.HasPrefix(att.Key, "tal:") || strings.HasPrefix(att.Key, "metal:") {
talAtts = append(talAtts, att)
} else {
originalAtts = append(originalAtts, att)
}
}
if len(talAtts) == 0 {
d.appendString("<")
d.append(tagName)
for _, att := range originalAtts {
d.appendString(" ")
d.appendString(att.Key)
d.appendString(`="`)
d.appendString(html.EscapeString(att.Val))
d.appendString(`"`)
}
d.appendString(">")
template.addRenderInstruction(d)
// Register an action to add the close tag in when we see it.
// This is done via an action so that we can use different logic for close tags that have tal commands
// tagName is captured by the closure
if !voidElement {
state.appendAction(getPlainEndTagAction(template, tagName))
} else {
// If we have a void element, pop it off the stack straight away
err = state.popTag(tagName)
if err != nil {
return nil, err
}
}
break
}
// Empty out the start and end tag state
state.talStartTag = &renderStartTag{tagName: tagName, originalAttributes: originalAtts, voidElement: voidElement}
state.talEndTag = &renderEndTag{tagName: tagName, checkOmitTagFlag: false}
// Sort the tal attributes into priority order
sort.Sort(talAttributes(talAtts))
// Process each one.
for _, talCommand := range talAtts {
properties, ok := talCommandProperties[talCommand.Key]
if !ok {
// As we are returning here we know that tokenizer will not get a chance to change the results of Raw() or Buffered()
return nil, newCompileError(ErrUnknownTalCommand, state.tokenizer.Raw(), state.tokenizer.Buffered())
}
err := properties.StartAction(originalAtts, talCommand.Val, state)
if err != nil {
return nil, err
}
}
// Output the start tag
currentStartTag := state.talStartTag
currentEndTag := state.talEndTag
template.addInstruction(state.talStartTag)
/*
Register the end tag function. This:
Updates the start tag with the location of the end tag
Does a special render for the end tag which know to omit itself if tal:omit is used
currentStartTag, currentEndTag and tagName are defined inside the for loop and so are captured within the closure
*/
state.insertAction(getTalEndTagAction(currentStartTag, currentEndTag, template))
/*
If we have a void element, run through all end actions immediately.
*/
if currentStartTag.voidElement {
err = state.popTag(tagName)
if err != nil {
return nil, err
}
}
case html.EndTagToken:
tagName, _ := tokenizer.TagName()
// WARNING: tagName is not immutable.
// For popTag this is not a problem, for other uses it may be.
// Pop a tag off our stack
err = state.popTag(tagName)
if err != nil {
return nil, err
}
//template.addRenderInstruction(d)
case html.SelfClosingTagToken:
case html.CommentToken:
var d buffer
d.appendString("<!--")
d.appendString(html.EscapeString(string(tokenizer.Text())))
d.appendString("-->")
template.addRenderInstruction(d)
case html.DoctypeToken:
var d buffer
d.appendString("<!DOCTYPE ")
d.appendString(html.EscapeString(string(tokenizer.Text())))
d.appendString(">")
template.addRenderInstruction(d)
}
}
return
}