// DispatchPOST updates the rss feed passed via post
func DispatchPOST(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
defer r.Body.Close()
if err := r.ParseForm(); err != nil {
c.Errorf("Error at in updatePost @ ParseForm. Error: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
dsFeed, err := data.FeedStored(c, r.FormValue("url"))
// get the url from the datastore
if err != nil {
c.Errorf("Error getting the url from the DS: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
if dsFeed == nil {
c.Errorf("Got no feed from the DS: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
// read the url
c.Infof("DS Feed @ importer: %v", dsFeed)
// parse feed
urls, err := getLinksFromFeed(dsFeed, r, c)
if err != nil {
c.Errorf("Error parsing the url: %v", dsFeed.URL)
dsFeed.Fails++
err := data.StoreFeed(c, *dsFeed)
if err != nil {
c.Errorf("Error while updating url %v in the DS: %v", dsFeed.URL, err)
}
w.WriteHeader(http.StatusInternalServerError)
return
}
c.Infof("URLs found in feed: %v", urls)
dsFeed.LastUpdate = time.Now() // update time of the last update
// put urls into kshare
for _, u := range urls {
importURL := config.KaffeeshareURL + "/k/share/json/" + dsFeed.Namespace //"test"
importURL += "/?url=" + url.QueryEscape(u)
c.Infof("URL we get %v", importURL)
_, body, err := extract.GetURL(importURL, r)
if err != nil {
c.Errorf("Error while importing url into namespace: %v, %v", importURL, err)
// we'll ignore the errors for now
// actually we may want to offload this into another task queue for easier retries
} else {
c.Infof("body: %v", string(body))
}
}
// update feed in DS
dsFeed.Fails = 0 // reset error count
err = data.StoreFeed(c, *dsFeed)
if err != nil {
c.Errorf("Error while updating url %v in the DS: %v", dsFeed.URL, err)
}
w.WriteHeader(http.StatusOK)
}
// DispatchPOST import the url passed via POST as a feed to the database
// and triggers an update if it was not in the DS before
func DispatchPOST(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
// get URL
r.ParseForm()
rssurl := r.Form.Get("rssurl")
if rssurl == "" {
w.WriteHeader(http.StatusInternalServerError)
return
}
if !(strings.HasPrefix(rssurl, "http://") || strings.HasPrefix(rssurl, "https://")) {
rssurl = "https://" + rssurl
}
// check if URL is valid
if !govalidator.IsRequestURL(rssurl) {
c.Errorf("Invalid URL. URL: %v", rssurl)
w.WriteHeader(http.StatusInternalServerError)
return
}
// check if URL is already in the datastore
if f, err := data.FeedStored(c, rssurl); err != nil {
c.Errorf("Error while checking if url is in DS. URL: %v, error: %v", rssurl, err)
w.WriteHeader(http.StatusInternalServerError)
return
} else if f != nil {
c.Infof("Feed is already in the DS %v", f)
// yes it is, lets marshal our reply and return it
b, err := json.Marshal(*f)
if err != nil {
c.Errorf("Error marshalling json: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Write(b)
return
}
// no it is not in the DS
// add the url to the datastore
feed := data.NewFeed(rssurl)
if err := data.StoreFeed(c, feed); err != nil {
c.Errorf("Error storing url in DS: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
// trigger import for that url
task := taskqueue.NewPOSTTask("/task/updater/post/", map[string][]string{"url": {feed.URL}})
if _, err := taskqueue.Add(c, task, "rss-feed-update"); err != nil {
c.Errorf("Error while triggering the url update: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
// return the data
b, err := json.Marshal(feed)
if err != nil {
c.Errorf("Error marshalling json: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Write(b)
}