// ParseQuery returns a mbox.Form from url.Values
func ParseQuery(query url.Values) *Form {
p := bluemonday.StrictPolicy()
form := new(Form)
additionalFields := ""
for k, v := range query {
k = strings.ToLower(k)
if k == "email" || k == "name" {
form.Email = v[0]
form.Email = p.Sanitize(form.Email)
} else if k == "subject" {
form.Subject = v[0]
form.Subject = p.Sanitize(form.Subject)
} else if k == "message" {
form.Message = k + ": " + v[0] + "<br>\n"
form.Message = p.Sanitize(form.Message)
} else if k != "cosgo" && k != "captchaid" && k != "captchasolution" {
additionalFields = additionalFields + k + ": " + v[0] + "<br>\n"
}
}
if form.Subject == "" || form.Subject == " " {
form.Subject = "[New Message]"
}
if additionalFields != "" {
if form.Message == "" {
form.Message = form.Message + "Message:\n<br>" + p.Sanitize(additionalFields)
} else {
form.Message = form.Message + "\n<br>Additional:\n<br>" + p.Sanitize(additionalFields)
}
}
return form
}
// ValidateInput checks the data input for correctness
func (m *UpdateTagModel) ValidateInput() (err error) {
if m.Ib == 0 {
return e.ErrInvalidParam
}
if m.TagType == 0 {
return e.ErrInvalidParam
}
// Initialize bluemonday
p := bluemonday.StrictPolicy()
// sanitize for html and xss
m.Tag = html.UnescapeString(p.Sanitize(m.Tag))
// Validate name input
tag := validate.Validate{Input: m.Tag, Max: config.Settings.Limits.TagMaxLength, Min: config.Settings.Limits.TagMinLength}
if tag.IsEmpty() {
return e.ErrNoTagName
} else if tag.MinPartsLength() {
return e.ErrTagShort
} else if tag.MaxLength() {
return e.ErrTagLong
}
return
}
//Excerpt returns post excerpt, 300 char long. Html tags are stripped.
func (post *Post) Excerpt() template.HTML {
//you can sanitize, cut it down, add images, etc
policy := bluemonday.StrictPolicy() //remove all html tags
sanitized := policy.Sanitize(string(blackfriday.MarkdownCommon([]byte(post.Content))))
excerpt := template.HTML(truncate(sanitized, 300) + "...")
return excerpt
}
func ExampleStrictPolicy() {
// StrictPolicy is equivalent to NewPolicy and as nothing else is declared
// we are stripping all elements (and their attributes)
p := bluemonday.StrictPolicy()
html := p.Sanitize(
`Goodbye <a onblur="alert(secret)" href="http://en.wikipedia.org/wiki/Goodbye_Cruel_World_(Pink_Floyd_song)">Cruel</a> World`,
)
fmt.Println(html)
// Output:
//Goodbye Cruel World
}
// ParseQueryGPG returns a mbox.Form from a url.Values but encodes the form.Message if publicKey is not nil
func ParseQueryGPG(query url.Values, publicKey []byte) *Form {
p := bluemonday.StrictPolicy()
form := new(Form)
additionalFields := ""
for k, v := range query {
k = strings.ToLower(k)
if k == "email" || k == "name" {
form.Email = v[0]
form.Email = p.Sanitize(form.Email)
} else if k == "subject" {
form.Subject = v[0]
form.Subject = p.Sanitize(form.Subject)
} else if k == "message" {
form.Message = k + ": " + v[0] + "<br>\n"
form.Message = p.Sanitize(form.Message)
} else if k != "cosgo" && k != "captchaid" && k != "captchasolution" {
additionalFields = additionalFields + k + ": " + v[0] + "<br>\n"
}
}
if form.Subject == "" || form.Subject == " " {
form.Subject = "[New Message]"
}
if additionalFields != "" {
if form.Message == "" {
form.Message = form.Message + "Message:\n<br>" + p.Sanitize(additionalFields)
} else {
form.Message = form.Message + "\n<br>Additional:\n<br>" + p.Sanitize(additionalFields)
}
}
if publicKey != nil {
tmpmsg, err := PGPEncode(form.Message, publicKey)
if err != nil {
log.Println("gpg error.")
log.Println(err)
} else {
form.Message = tmpmsg
}
}
return form
}
func main() {
// setup command flag
var tag string
var maxPage int
flag.StringVar(&tag, "tag", DEFAULT_TAG, "specified tag to scrape")
flag.IntVar(&maxPage, "maxpage", DEFAULT_MAX_PAGE, "the maximum number of scraped page")
flag.Parse()
if tag == "" {
fmt.Printf("tag is not specied. default value '%s' is used.\n", DEFAULT_TAG)
}
if maxPage == 0 {
fmt.Printf("maximum page is not specied. default value %d is used.\n", DEFAULT_MAX_PAGE)
}
// prepare for id regex
idRegex := regexp.MustCompile(`\d{6,7}`)
// create sanitizer
sanitizer := bluemonday.StrictPolicy()
for i := 1; i <= maxPage; i++ {
url := fmt.Sprintf("http://www.liputan6.com/tag/%s?type=text&page=%d", tag, i)
doc, err := goquery.NewDocument(url)
if err != nil {
fmt.Printf("error page %d; %s\n", i, err)
continue
}
// for each link in the page, print the title & link
doc.Find("a.articles--rows--item__title-link").Each(func(i int, s *goquery.Selection) {
// get article URL
articleURL, exists := s.Attr("href")
if !exists {
fmt.Println("error: article url not exists")
return
}
// get article id from URL
articleID := idRegex.FindString(articleURL)
if articleID == "" {
fmt.Printf("error: article id not found in %s\n", articleURL)
return
}
// fetch article content
article, err := goquery.NewDocument(articleURL)
if err != nil {
fmt.Printf("error: couldn't create new document from %s\n", articleURL)
return
}
title := article.Find("h1.read-page--header__title").First().Text()
content := sanitizer.Sanitize(article.Find("div.read-page__content-body").First().Text())
// create new file
filename := fmt.Sprintf("liputan6.%s.%s", tag, articleID)
f, err := os.Create(filename)
if err != nil {
fmt.Printf("error: couldn't create a file %s\n", filename)
return
}
_, err = f.WriteString(title + "\n" + content)
if err != nil {
fmt.Printf("error: couldn't write to a file %s\n", filename)
return
}
err = f.Close()
if err != nil {
fmt.Printf("error: couldn't close the file %s\n", filename)
return
}
})
}
}
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"TRACE",
"OPTIONS",
"CONNECT",
}
var empty = struct{}{}
var router = mux.NewRouter()
var isRouterActive = false
var sanitizer = bluemonday.StrictPolicy()
var geoipdb, _ = geoip2.Open("GeoIP2-Country.mmdb")
func geolocate(h HandlerArgs) (string, string) {
return geolocateIp(h.Request.RemoteAddr)
}
func geolocateIp(ip string) (string, string) {
record, err := geoipdb.Country(net.ParseIP(ip))
if err != nil {
return "", ""
}
country := strings.ToLower(record.Country.IsoCode)
continent := strings.ToLower(record.Continent.Code)
func init() {
strict = bluemonday.StrictPolicy()
}
func init() {
strictPolicy = bluemonday.StrictPolicy()
normalPolicy = bluemonday.UGCPolicy()
}