Exemple #1
0
// DispatchPOST updates the rss feed passed via post
func DispatchPOST(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	defer r.Body.Close()

	if err := r.ParseForm(); err != nil {
		c.Errorf("Error at in updatePost @ ParseForm. Error: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	dsFeed, err := data.FeedStored(c, r.FormValue("url"))

	// get the url from the datastore
	if err != nil {
		c.Errorf("Error getting the url from the DS: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if dsFeed == nil {
		c.Errorf("Got no feed from the DS: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	importURL := config.KaffeeshareURL + "/k/share/json/" + dsFeed.Namespace
	importURL += "/?url=" + url.QueryEscape("https://kaffeebot.com/stops/updating/goodbye")
	c.Infof("URL we get %v", importURL)
	_, body, err := extract.GetURL(importURL, r)
	if err != nil {
		c.Errorf("Error while importing url into namespace: %v, %v", importURL, err)
		// we'll ignore the errors for now
		// actually we may want to offload this into another task queue for easier retries
	} else {
		c.Infof("body: %v", string(body))
	}

	w.WriteHeader(http.StatusOK)
}
Exemple #2
0
// DispatchPOST updates the rss feed passed via post
func DispatchPOST(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	defer r.Body.Close()

	if err := r.ParseForm(); err != nil {
		c.Errorf("Error at in updatePost @ ParseForm. Error: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	dsFeed, err := data.FeedStored(c, r.FormValue("url"))

	// get the url from the datastore
	if err != nil {
		c.Errorf("Error getting the url from the DS: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if dsFeed == nil {
		c.Errorf("Got no feed from the DS: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// read the url
	c.Infof("DS Feed @ importer: %v", dsFeed)

	// parse feed
	urls, err := getLinksFromFeed(dsFeed, r, c)
	if err != nil {
		c.Errorf("Error parsing the url: %v", dsFeed.URL)
		dsFeed.Fails++
		err := data.StoreFeed(c, *dsFeed)
		if err != nil {
			c.Errorf("Error while updating url %v in the DS: %v", dsFeed.URL, err)
		}

		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	c.Infof("URLs found in feed: %v", urls)

	dsFeed.LastUpdate = time.Now() // update time of the last update

	// put urls into kshare
	for _, u := range urls {
		importURL := config.KaffeeshareURL + "/k/share/json/" + dsFeed.Namespace //"test"
		importURL += "/?url=" + url.QueryEscape(u)
		c.Infof("URL we get %v", importURL)
		_, body, err := extract.GetURL(importURL, r)
		if err != nil {
			c.Errorf("Error while importing url into namespace: %v, %v", importURL, err)
			// we'll ignore the errors for now
			// actually we may want to offload this into another task queue for easier retries
		} else {
			c.Infof("body: %v", string(body))
		}
	}

	// update feed in DS
	dsFeed.Fails = 0 // reset error count

	err = data.StoreFeed(c, *dsFeed)
	if err != nil {
		c.Errorf("Error while updating url %v in the DS: %v", dsFeed.URL, err)
	}
	w.WriteHeader(http.StatusOK)
}
Exemple #3
0
// DispatchPOST import the url passed via POST as a feed to the database
// and triggers an update if it was not in the DS before
func DispatchPOST(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// get URL
	r.ParseForm()
	rssurl := r.Form.Get("rssurl")

	if rssurl == "" {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if !(strings.HasPrefix(rssurl, "http://") || strings.HasPrefix(rssurl, "https://")) {
		rssurl = "https://" + rssurl
	}

	// check if URL is valid
	if !govalidator.IsRequestURL(rssurl) {
		c.Errorf("Invalid URL. URL: %v", rssurl)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// check if URL is already in the datastore
	if f, err := data.FeedStored(c, rssurl); err != nil {
		c.Errorf("Error while checking if url is in DS. URL: %v, error: %v", rssurl, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	} else if f != nil {
		c.Infof("Feed is already in the DS %v", f)
		// yes it is, lets marshal our reply and return it
		b, err := json.Marshal(*f)
		if err != nil {
			c.Errorf("Error marshalling json: %v", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.Write(b)
		return
	}

	// no it is not in the DS
	// add the url to the datastore
	feed := data.NewFeed(rssurl)

	if err := data.StoreFeed(c, feed); err != nil {
		c.Errorf("Error storing url in DS: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// trigger import for that url
	task := taskqueue.NewPOSTTask("/task/updater/post/", map[string][]string{"url": {feed.URL}})
	if _, err := taskqueue.Add(c, task, "rss-feed-update"); err != nil {
		c.Errorf("Error while triggering the url update: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// return the data
	b, err := json.Marshal(feed)
	if err != nil {
		c.Errorf("Error marshalling json: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(b)
}