Esempio n. 1
0
// ogtags extracts the og:title, og:image, ... tags from a webpage
func defaultHTML(i *data.Item, sourceURL string, doc *goquery.Document) {
	fmt.Println("Running OG extract.")

	selection := doc.Find("title")
	if len(selection.Nodes) != 0 {
		i.Caption = selection.Nodes[0].FirstChild.Data
	}

	selection = doc.Find("meta[property*='og']")

	for _, e := range selection.Nodes {
		m := htmlAttributeToMap(e.Attr)

		if m["property"] == "og:title" {
			i.Caption = m["content"]
		}
		if m["property"] == "og:image" {
			if !govalidator.IsRequestURL(m["content"]) {
				log.Println("Invalid url in og:image. " + sourceURL)
				continue
			}
			i.ImageURL = m["content"]
		}
		if m["property"] == "og:url" {
			if !govalidator.IsRequestURL(m["content"]) {
				log.Println("Invalid url in og:url. " + sourceURL)
				continue
			}
			i.URL = m["content"]
		}
		if m["property"] == "og:description" {
			i.Description = m["content"]
		}
	}
}
Esempio n. 2
0
func garfield(i *data.Item, sourceURL string, doc *goquery.Document) {
	if !strings.Contains(sourceURL, "www.gocomics.com/garfield") {
		return
	}

	fmt.Println("Running Garfield plugin.")

	// update title

	selection := doc.Find(".strip")
	if len(selection.Nodes) == 0 {
		fmt.Println("Garfield plugin found no .strip. " + sourceURL)
	} else {
		if len(selection.Nodes) > 1 {
			fmt.Println("Garfield plugin found >1 .strip. " + sourceURL)
		}
		m := htmlAttributeToMap(selection.Nodes[0].Attr)

		if govalidator.IsRequestURL(m["src"]) {
			i.Description = "<img src =\""
			i.Description += m["src"]
			i.Description += "\" />"
		} else {
			fmt.Println("Amazon plugin invalid url. " + m["src"])
		}
		i.ImageURL = ""
	}

}
Esempio n. 3
0
// DispatchJSON receives an extension json request
func DispatchJSON(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// get namespace
	namespace := mux.Vars(r)["namespace"]
	if namespace == "" {
		startpage.Dispatch(w, r)
		return
	}

	shareURL := r.URL.Query().Get("url")

	if !govalidator.IsRequestURL(shareURL) {
		c.Errorf("Error at unmarshalling for share/json. Namespace: %v. Error: %v", namespace, shareURL)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	i := extract.ItemFromURL(shareURL, r)
	i.Namespace = namespace

	c.Infof("Item: %v", i)

	if err := data.StoreItem(c, i); err != nil {
		c.Errorf("Error at in StoreItem. Item: %v. Error: %v", i, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Write(statusOk)
}
Esempio n. 4
0
func validateChartSources(cf *chart.Metadata) error {
	for _, source := range cf.Sources {
		if source == "" || !govalidator.IsRequestURL(source) {
			return fmt.Errorf("invalid source URL '%s'", source)
		}
	}
	return nil
}
Esempio n. 5
0
func validateChartSources(cf *chart.Metadata) (lintError support.LintError) {
	for _, source := range cf.Sources {
		if source == "" || !govalidator.IsRequestURL(source) {
			lintError = fmt.Errorf("Chart.yaml: 'source' invalid URL %s", source)
		}
	}
	return
}
Esempio n. 6
0
// DispatchEmail parses incoming emails
func DispatchEmail(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	defer r.Body.Close()

	msg, err := mail.ReadMessage(r.Body)
	if err != nil {
		c.Errorf("Error at mail.ReadMessage in DispatchEmail. Error: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	c.Infof("header: %v", msg.Header)

	// get namespaces
	namespaces, err := getNamespaces(msg)
	if err != nil {
		c.Errorf("Error at parsing the receiver fields. Error: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	c.Infof("Detected namespaces: %v", namespaces)

	// get body
	body, err := extractBody(c, msg.Header, msg.Body)
	if err != nil {
		c.Errorf("Error at parsing the body. Error: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	c.Infof("Received mail: %v", body)

	urls, err := parseBody(c, body)
	c.Infof("Found urls: %v", urls)

	for _, shareURL := range urls {
		if !govalidator.IsRequestURL(shareURL) {
			c.Errorf("Invalid URL. Error: %v", shareURL)
			continue
		}

		i := extract.ItemFromURL(shareURL, r)

		for _, namespace := range namespaces {
			i.Namespace = namespace
			c.Infof("Item: %v", i)

			if err := data.StoreItem(c, i); err != nil {
				c.Errorf("Error at in StoreItem. Item: %v. Error: %v", i, err)
				continue
			}
		}
	}

}
Esempio n. 7
0
// IsValidRedirectURI validates a redirect_uri as specified in:
//
// * https://tools.ietf.org/html/rfc6749#section-3.1.2
//   * The redirection endpoint URI MUST be an absolute URI as defined by [RFC3986] Section 4.3.
//   * The endpoint URI MUST NOT include a fragment component.
// * https://tools.ietf.org/html/rfc3986#section-4.3
//   absolute-URI  = scheme ":" hier-part [ "?" query ]
// * https://tools.ietf.org/html/rfc6819#section-5.1.1
func IsValidRedirectURI(redirectURI *url.URL) bool {
	// We need to explicitly check for a scheme
	if !govalidator.IsRequestURL(redirectURI.String()) {
		return false
	}

	if redirectURI.Fragment != "" {
		// "The endpoint URI MUST NOT include a fragment component."
		return false
	}

	return true
}
Esempio n. 8
0
// Validate does some validation to be able to store the record.
func (u *Registry) Validate(db *gorm.DB) {
	if !govalidator.StringLength(u.Name, "1", "255") {
		db.AddError(fmt.Errorf("Name should be longer than 1 and shorter than 255"))
	}

	if u.Name != "" {
		notFound := db.Where(
			"name = ?",
			u.Name,
		).Not(
			"id",
			u.ID,
		).First(
			&Registry{},
		).RecordNotFound()

		if !notFound {
			db.AddError(fmt.Errorf("Name is already present"))
		}
	}

	if !govalidator.StringLength(u.Host, "1", "255") {
		db.AddError(fmt.Errorf("Host should be longer than 1 and shorter than 255"))
	}

	if !govalidator.IsRequestURL(u.Host) {
		db.AddError(fmt.Errorf(
			"Host must be a valid URL",
		))
	}

	if u.Host != "" {
		notFound := db.Where(
			"host = ?",
			u.Host,
		).Not(
			"id",
			u.ID,
		).First(
			&Registry{},
		).RecordNotFound()

		if !notFound {
			db.AddError(fmt.Errorf("Host is already present"))
		}
	}
}
Esempio n. 9
0
func main() {
	webaddr := flag.String("url", "", "The URL to be parsed.")
	flag.Parse()

	// Check if URL was passed at command line
	if *webaddr == "" {
		flag.PrintDefaults()
		return
	}

	// Is it a valid URL?
	if !govalidator.IsRequestURL(*webaddr) {
		fmt.Println("Invalid URL")
		return
	}

	fmt.Print(extract.ItemFromURL(*webaddr, nil))
}
Esempio n. 10
0
func dilbert(i *data.Item, sourceURL string, doc *goquery.Document) {
	if !(strings.Contains(sourceURL, "feed.dilbert.com/") ||
		strings.Contains(sourceURL, "dilbert.com/strips/")) {
		return
	}

	fmt.Println("Running Dilbert plugin.")

	selection := doc.Find(".STR_Image").Find("img")

	if len(selection.Nodes) == 0 {
		fmt.Println("Dilbert plugin found no .STR_Image. " + sourceURL)
		return
	}

	if len(selection.Nodes) > 1 {
		fmt.Println("Dilbert plugin found >1 .STR_Image. " + sourceURL)
	}

	e := selection.Nodes[0]
	if e.Type == html.ElementNode && e.Data == "img" {
		m := htmlAttributeToMap(e.Attr)
		u := ""
		if !strings.Contains(m["src"], "://dilbert.com") {
			u += "https://dilbert.com"
		}

		u += m["src"]
		if govalidator.IsRequestURL(u) {
			i.Description = "<img src=\""
			i.Description += u
			i.Description += "\" />"
		} else {
			fmt.Println("Dilbert plugin invalid url. " + u)
		}

	} else {
		fmt.Println("Dilbert plugin no image tag where we expect one.")
		fmt.Println(e)
	}

	i.ImageURL = ""
	i.Caption = "Dilbert"
}
Esempio n. 11
0
func (cnf *Config) Validate() error {
	if !govalidator.IsRequestURL(cnf.Host) {
		return errors.New("invalid host url")
	}

	if cnf.Repeat == 0 ||
		cnf.Prefix == "" ||
		cnf.User == "" ||
		cnf.Group == "" ||
		cnf.LogLevel == "" {
		return errors.New("invalid runtime options")
	}

	if len(cnf.Sets) == 0 {
		return errors.New("no template sets found")
	}

	return nil
}
Esempio n. 12
0
func validateChartHome(cf *chart.Metadata) error {
	if cf.Home != "" && !govalidator.IsRequestURL(cf.Home) {
		return fmt.Errorf("invalid home URL '%s'", cf.Home)
	}
	return nil
}
Esempio n. 13
0
func validateChartHome(cf *chart.Metadata) (lintError support.LintError) {
	if cf.Home != "" && !govalidator.IsRequestURL(cf.Home) {
		lintError = fmt.Errorf("Chart.yaml: 'home' invalid URL %s", cf.Home)
	}
	return
}
Esempio n. 14
0
// DispatchPOST import the url passed via POST as a feed to the database
// and triggers an update if it was not in the DS before
func DispatchPOST(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// get URL
	r.ParseForm()
	rssurl := r.Form.Get("rssurl")

	if rssurl == "" {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if !(strings.HasPrefix(rssurl, "http://") || strings.HasPrefix(rssurl, "https://")) {
		rssurl = "https://" + rssurl
	}

	// check if URL is valid
	if !govalidator.IsRequestURL(rssurl) {
		c.Errorf("Invalid URL. URL: %v", rssurl)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// check if URL is already in the datastore
	if f, err := data.FeedStored(c, rssurl); err != nil {
		c.Errorf("Error while checking if url is in DS. URL: %v, error: %v", rssurl, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	} else if f != nil {
		c.Infof("Feed is already in the DS %v", f)
		// yes it is, lets marshal our reply and return it
		b, err := json.Marshal(*f)
		if err != nil {
			c.Errorf("Error marshalling json: %v", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.Write(b)
		return
	}

	// no it is not in the DS
	// add the url to the datastore
	feed := data.NewFeed(rssurl)

	if err := data.StoreFeed(c, feed); err != nil {
		c.Errorf("Error storing url in DS: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// trigger import for that url
	task := taskqueue.NewPOSTTask("/task/updater/post/", map[string][]string{"url": {feed.URL}})
	if _, err := taskqueue.Add(c, task, "rss-feed-update"); err != nil {
		c.Errorf("Error while triggering the url update: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// return the data
	b, err := json.Marshal(feed)
	if err != nil {
		c.Errorf("Error marshalling json: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(b)
}
Esempio n. 15
0
func (i IsRequestURLChecker) IsFormat(data string) bool {
	return govalidator.IsRequestURL(data)
}
Esempio n. 16
0
func (url Url) Validate() error {
	if !govalidator.IsRequestURL(url.String()) {
		return ErrInvalidUrl
	}
	return nil
}
Esempio n. 17
0
func amazon(i *data.Item, sourceURL string, doc *goquery.Document) {
	if !strings.Contains(sourceURL, "www.amazon.") {
		return
	}

	fmt.Println("Running Amazon plugin.")

	// find picture
	{
		selection := doc.Find("#landingImage")
		if len(selection.Nodes) == 0 {
			fmt.Println("Amazon plugin found no #landingImage. " + sourceURL)
		} else {
			if len(selection.Nodes) > 1 {
				fmt.Println("Amazon plugin found >1 #landingImage. " + sourceURL)
			}
			for _, e := range selection.Nodes {
				if e.Type == html.ElementNode && e.Data == "img" {
					m := htmlAttributeToMap(e.Attr)
					if govalidator.IsRequestURL(m["data-old-hires"]) {
						i.ImageURL = m["data-old-hires"]
					} else {
						fmt.Println("Amazon plugin imgURL invalid. " + m["data-old-hires"])
					}
				}
			}
		}
	}

	// update url to contain tag
	{
		// This is our tag. We should make it configurable
		urlExtension := "tag=" + "gschaftshuonl-21"
		start := strings.Index(i.URL, "tag=")
		if start != -1 {
			end := strings.Index(i.URL[start+1:], "&") + start + 1
			i.URL = i.URL[:start] + i.URL[end:]
		}

		if strings.Index(i.URL, "?") == -1 {
			i.URL += "?" + urlExtension
		} else {
			i.URL += "&" + urlExtension
		}
	}

	// update title
	{
		selection := doc.Find("#productTitle")
		if len(selection.Nodes) == 0 {
			fmt.Println("Amazon plugin found no #productTitle. " + sourceURL)
		} else {
			if len(selection.Nodes) > 1 {
				fmt.Println("Amazon plugin found >1 #productTitle. " + sourceURL)
			}
			for _, e := range selection.Nodes {
				if e.Type == html.ElementNode && e.Data == "span" {
					i.Caption = e.FirstChild.Data
				}
			}
		}

	}
}