// ogtags extracts the og:title, og:image, ... tags from a webpage func defaultHTML(i *data.Item, sourceURL string, doc *goquery.Document) { fmt.Println("Running OG extract.") selection := doc.Find("title") if len(selection.Nodes) != 0 { i.Caption = selection.Nodes[0].FirstChild.Data } selection = doc.Find("meta[property*='og']") for _, e := range selection.Nodes { m := htmlAttributeToMap(e.Attr) if m["property"] == "og:title" { i.Caption = m["content"] } if m["property"] == "og:image" { if !govalidator.IsRequestURL(m["content"]) { log.Println("Invalid url in og:image. " + sourceURL) continue } i.ImageURL = m["content"] } if m["property"] == "og:url" { if !govalidator.IsRequestURL(m["content"]) { log.Println("Invalid url in og:url. " + sourceURL) continue } i.URL = m["content"] } if m["property"] == "og:description" { i.Description = m["content"] } } }
func garfield(i *data.Item, sourceURL string, doc *goquery.Document) { if !strings.Contains(sourceURL, "www.gocomics.com/garfield") { return } fmt.Println("Running Garfield plugin.") // update title selection := doc.Find(".strip") if len(selection.Nodes) == 0 { fmt.Println("Garfield plugin found no .strip. " + sourceURL) } else { if len(selection.Nodes) > 1 { fmt.Println("Garfield plugin found >1 .strip. " + sourceURL) } m := htmlAttributeToMap(selection.Nodes[0].Attr) if govalidator.IsRequestURL(m["src"]) { i.Description = "<img src =\"" i.Description += m["src"] i.Description += "\" />" } else { fmt.Println("Amazon plugin invalid url. " + m["src"]) } i.ImageURL = "" } }
// DispatchJSON receives an extension json request func DispatchJSON(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) // get namespace namespace := mux.Vars(r)["namespace"] if namespace == "" { startpage.Dispatch(w, r) return } shareURL := r.URL.Query().Get("url") if !govalidator.IsRequestURL(shareURL) { c.Errorf("Error at unmarshalling for share/json. Namespace: %v. Error: %v", namespace, shareURL) w.WriteHeader(http.StatusInternalServerError) return } i := extract.ItemFromURL(shareURL, r) i.Namespace = namespace c.Infof("Item: %v", i) if err := data.StoreItem(c, i); err != nil { c.Errorf("Error at in StoreItem. Item: %v. Error: %v", i, err) w.WriteHeader(http.StatusInternalServerError) return } w.Write(statusOk) }
func validateChartSources(cf *chart.Metadata) error { for _, source := range cf.Sources { if source == "" || !govalidator.IsRequestURL(source) { return fmt.Errorf("invalid source URL '%s'", source) } } return nil }
func validateChartSources(cf *chart.Metadata) (lintError support.LintError) { for _, source := range cf.Sources { if source == "" || !govalidator.IsRequestURL(source) { lintError = fmt.Errorf("Chart.yaml: 'source' invalid URL %s", source) } } return }
// DispatchEmail parses incoming emails func DispatchEmail(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) defer r.Body.Close() msg, err := mail.ReadMessage(r.Body) if err != nil { c.Errorf("Error at mail.ReadMessage in DispatchEmail. Error: %v", err) w.WriteHeader(http.StatusInternalServerError) return } c.Infof("header: %v", msg.Header) // get namespaces namespaces, err := getNamespaces(msg) if err != nil { c.Errorf("Error at parsing the receiver fields. Error: %v", err) w.WriteHeader(http.StatusInternalServerError) return } c.Infof("Detected namespaces: %v", namespaces) // get body body, err := extractBody(c, msg.Header, msg.Body) if err != nil { c.Errorf("Error at parsing the body. Error: %v", err) w.WriteHeader(http.StatusInternalServerError) return } c.Infof("Received mail: %v", body) urls, err := parseBody(c, body) c.Infof("Found urls: %v", urls) for _, shareURL := range urls { if !govalidator.IsRequestURL(shareURL) { c.Errorf("Invalid URL. Error: %v", shareURL) continue } i := extract.ItemFromURL(shareURL, r) for _, namespace := range namespaces { i.Namespace = namespace c.Infof("Item: %v", i) if err := data.StoreItem(c, i); err != nil { c.Errorf("Error at in StoreItem. Item: %v. Error: %v", i, err) continue } } } }
// IsValidRedirectURI validates a redirect_uri as specified in: // // * https://tools.ietf.org/html/rfc6749#section-3.1.2 // * The redirection endpoint URI MUST be an absolute URI as defined by [RFC3986] Section 4.3. // * The endpoint URI MUST NOT include a fragment component. // * https://tools.ietf.org/html/rfc3986#section-4.3 // absolute-URI = scheme ":" hier-part [ "?" query ] // * https://tools.ietf.org/html/rfc6819#section-5.1.1 func IsValidRedirectURI(redirectURI *url.URL) bool { // We need to explicitly check for a scheme if !govalidator.IsRequestURL(redirectURI.String()) { return false } if redirectURI.Fragment != "" { // "The endpoint URI MUST NOT include a fragment component." return false } return true }
// Validate does some validation to be able to store the record. func (u *Registry) Validate(db *gorm.DB) { if !govalidator.StringLength(u.Name, "1", "255") { db.AddError(fmt.Errorf("Name should be longer than 1 and shorter than 255")) } if u.Name != "" { notFound := db.Where( "name = ?", u.Name, ).Not( "id", u.ID, ).First( &Registry{}, ).RecordNotFound() if !notFound { db.AddError(fmt.Errorf("Name is already present")) } } if !govalidator.StringLength(u.Host, "1", "255") { db.AddError(fmt.Errorf("Host should be longer than 1 and shorter than 255")) } if !govalidator.IsRequestURL(u.Host) { db.AddError(fmt.Errorf( "Host must be a valid URL", )) } if u.Host != "" { notFound := db.Where( "host = ?", u.Host, ).Not( "id", u.ID, ).First( &Registry{}, ).RecordNotFound() if !notFound { db.AddError(fmt.Errorf("Host is already present")) } } }
func main() { webaddr := flag.String("url", "", "The URL to be parsed.") flag.Parse() // Check if URL was passed at command line if *webaddr == "" { flag.PrintDefaults() return } // Is it a valid URL? if !govalidator.IsRequestURL(*webaddr) { fmt.Println("Invalid URL") return } fmt.Print(extract.ItemFromURL(*webaddr, nil)) }
func dilbert(i *data.Item, sourceURL string, doc *goquery.Document) { if !(strings.Contains(sourceURL, "feed.dilbert.com/") || strings.Contains(sourceURL, "dilbert.com/strips/")) { return } fmt.Println("Running Dilbert plugin.") selection := doc.Find(".STR_Image").Find("img") if len(selection.Nodes) == 0 { fmt.Println("Dilbert plugin found no .STR_Image. " + sourceURL) return } if len(selection.Nodes) > 1 { fmt.Println("Dilbert plugin found >1 .STR_Image. " + sourceURL) } e := selection.Nodes[0] if e.Type == html.ElementNode && e.Data == "img" { m := htmlAttributeToMap(e.Attr) u := "" if !strings.Contains(m["src"], "://dilbert.com") { u += "https://dilbert.com" } u += m["src"] if govalidator.IsRequestURL(u) { i.Description = "<img src=\"" i.Description += u i.Description += "\" />" } else { fmt.Println("Dilbert plugin invalid url. " + u) } } else { fmt.Println("Dilbert plugin no image tag where we expect one.") fmt.Println(e) } i.ImageURL = "" i.Caption = "Dilbert" }
func (cnf *Config) Validate() error { if !govalidator.IsRequestURL(cnf.Host) { return errors.New("invalid host url") } if cnf.Repeat == 0 || cnf.Prefix == "" || cnf.User == "" || cnf.Group == "" || cnf.LogLevel == "" { return errors.New("invalid runtime options") } if len(cnf.Sets) == 0 { return errors.New("no template sets found") } return nil }
func validateChartHome(cf *chart.Metadata) error { if cf.Home != "" && !govalidator.IsRequestURL(cf.Home) { return fmt.Errorf("invalid home URL '%s'", cf.Home) } return nil }
func validateChartHome(cf *chart.Metadata) (lintError support.LintError) { if cf.Home != "" && !govalidator.IsRequestURL(cf.Home) { lintError = fmt.Errorf("Chart.yaml: 'home' invalid URL %s", cf.Home) } return }
// DispatchPOST import the url passed via POST as a feed to the database // and triggers an update if it was not in the DS before func DispatchPOST(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) // get URL r.ParseForm() rssurl := r.Form.Get("rssurl") if rssurl == "" { w.WriteHeader(http.StatusInternalServerError) return } if !(strings.HasPrefix(rssurl, "http://") || strings.HasPrefix(rssurl, "https://")) { rssurl = "https://" + rssurl } // check if URL is valid if !govalidator.IsRequestURL(rssurl) { c.Errorf("Invalid URL. URL: %v", rssurl) w.WriteHeader(http.StatusInternalServerError) return } // check if URL is already in the datastore if f, err := data.FeedStored(c, rssurl); err != nil { c.Errorf("Error while checking if url is in DS. URL: %v, error: %v", rssurl, err) w.WriteHeader(http.StatusInternalServerError) return } else if f != nil { c.Infof("Feed is already in the DS %v", f) // yes it is, lets marshal our reply and return it b, err := json.Marshal(*f) if err != nil { c.Errorf("Error marshalling json: %v", err) w.WriteHeader(http.StatusInternalServerError) return } w.Write(b) return } // no it is not in the DS // add the url to the datastore feed := data.NewFeed(rssurl) if err := data.StoreFeed(c, feed); err != nil { c.Errorf("Error storing url in DS: %v", err) w.WriteHeader(http.StatusInternalServerError) return } // trigger import for that url task := taskqueue.NewPOSTTask("/task/updater/post/", map[string][]string{"url": {feed.URL}}) if _, err := taskqueue.Add(c, task, "rss-feed-update"); err != nil { c.Errorf("Error while triggering the url update: %v", err) w.WriteHeader(http.StatusInternalServerError) return } // return the data b, err := json.Marshal(feed) if err != nil { c.Errorf("Error marshalling json: %v", err) w.WriteHeader(http.StatusInternalServerError) return } w.Write(b) }
func (i IsRequestURLChecker) IsFormat(data string) bool { return govalidator.IsRequestURL(data) }
func (url Url) Validate() error { if !govalidator.IsRequestURL(url.String()) { return ErrInvalidUrl } return nil }
func amazon(i *data.Item, sourceURL string, doc *goquery.Document) { if !strings.Contains(sourceURL, "www.amazon.") { return } fmt.Println("Running Amazon plugin.") // find picture { selection := doc.Find("#landingImage") if len(selection.Nodes) == 0 { fmt.Println("Amazon plugin found no #landingImage. " + sourceURL) } else { if len(selection.Nodes) > 1 { fmt.Println("Amazon plugin found >1 #landingImage. " + sourceURL) } for _, e := range selection.Nodes { if e.Type == html.ElementNode && e.Data == "img" { m := htmlAttributeToMap(e.Attr) if govalidator.IsRequestURL(m["data-old-hires"]) { i.ImageURL = m["data-old-hires"] } else { fmt.Println("Amazon plugin imgURL invalid. " + m["data-old-hires"]) } } } } } // update url to contain tag { // This is our tag. We should make it configurable urlExtension := "tag=" + "gschaftshuonl-21" start := strings.Index(i.URL, "tag=") if start != -1 { end := strings.Index(i.URL[start+1:], "&") + start + 1 i.URL = i.URL[:start] + i.URL[end:] } if strings.Index(i.URL, "?") == -1 { i.URL += "?" + urlExtension } else { i.URL += "&" + urlExtension } } // update title { selection := doc.Find("#productTitle") if len(selection.Nodes) == 0 { fmt.Println("Amazon plugin found no #productTitle. " + sourceURL) } else { if len(selection.Nodes) > 1 { fmt.Println("Amazon plugin found >1 #productTitle. " + sourceURL) } for _, e := range selection.Nodes { if e.Type == html.ElementNode && e.Data == "span" { i.Caption = e.FirstChild.Data } } } } }