func Main(c mpg.Context, w http.ResponseWriter, r *http.Request) { if err := templates.ExecuteTemplate(w, "base.html", includes(c, w, r)); err != nil { c.Errorf("%v", err) serveError(w, err) return } }
func CFix(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Infof("fix feed %s", url) f := Feed{Url: url} if err := gn.Get(&f); err != nil { c.Criticalf("cfix err: %v", err) serveError(w, err) return } q := datastore.NewQuery("S").Ancestor(gn.Key(&f)) var ss []*Story keys, err := q.GetAll(c, &ss) if err != nil { c.Errorf("getall err: %v", err) serveError(w, err) return } c.Infof("trying to fix %v stories", len(ss)) const putLimit = 500 for i := 0; i <= len(keys)/putLimit; i++ { lo := i * putLimit hi := (i + 1) * putLimit if hi > len(keys) { hi = len(keys) } c.Infof("%v - %v", lo, hi) if _, err := datastore.PutMulti(c, keys[lo:hi], ss[lo:hi]); err != nil { c.Errorf("err: %v, %v, %v", lo, hi, err) } } }
func AddSubscription(c mpg.Context, w http.ResponseWriter, r *http.Request) { backupOPML(c) cu := user.Current(c) url := r.FormValue("url") o := &OpmlOutline{ Outline: []*OpmlOutline{ &OpmlOutline{XmlUrl: url}, }, } if err := addFeed(c, cu.ID, o); err != nil { c.Errorf("add sub error (%s): %s", url, err.Error()) serveError(w, err) return } gn := goon.FromContext(c) ud := UserData{Id: "data", Parent: gn.Key(&User{Id: cu.ID})} gn.Get(&ud) if err := mergeUserOpml(c, &ud, o); err != nil { c.Errorf("add sub error opml (%v): %v", url, err) serveError(w, err) return } gn.PutMany(&ud, &Log{ Parent: ud.Parent, Id: time.Now().UnixNano(), Text: fmt.Sprintf("add sub: %v", url), }) if r.Method == "GET" { http.Redirect(w, r, routeUrl("main"), http.StatusFound) } backupOPML(c) }
func Charge(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} uc := UserCharge{Id: 1, Parent: gn.Key(&u)} if err := gn.Get(&u); err != nil { serveError(w, err) return } else if u.Account != AFree { serveError(w, fmt.Errorf("You're already subscribed.")) return } if err := gn.Get(&uc); err == nil && len(uc.Customer) > 0 { serveError(w, fmt.Errorf("You're already subscribed.")) return } else if err != datastore.ErrNoSuchEntity { serveError(w, err) return } resp, err := stripe(c, "POST", "customers", url.Values{ "email": {u.Email}, "description": {u.Id}, "card": {r.FormValue("stripeToken")}, "plan": {STRIPE_PLAN}, }.Encode()) if err != nil { serveError(w, err) return } else if resp.StatusCode != http.StatusOK { c.Errorf("%s", resp.Body) serveError(w, fmt.Errorf("Error")) return } defer resp.Body.Close() b, _ := ioutil.ReadAll(resp.Body) var sc StripeCustomer if err := json.Unmarshal(b, &sc); err != nil { serveError(w, err) return } if err := gn.RunInTransaction(func(gn *goon.Goon) error { if err := gn.Get(&u); err != nil && err != datastore.ErrNoSuchEntity { return err } if err := gn.Get(&uc); err != nil && err != datastore.ErrNoSuchEntity { return err } u.Account = APaid uc.Customer = sc.Id uc.Last4 = sc.Card.Last4 uc.Created = time.Unix(sc.Created, 0) _, err := gn.PutMany(&u, &uc) return err }, nil); err != nil { serveError(w, err) return } b, _ = json.Marshal(&uc) w.Write(b) }
func doUncheckout(c mpg.Context) (*UserCharge, error) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} uc := UserCharge{Id: 1, Parent: gn.Key(&u)} if err := gn.Get(&u); err != nil { return nil, err } if err := gn.Get(&uc); err != nil || len(uc.Customer) == 0 { return nil, err } resp, err := stripe(c, "DELETE", "customers/"+uc.Customer, "") if err != nil { return nil, err } else if resp.StatusCode != http.StatusOK { c.Errorf("%s", resp.Body) c.Errorf("stripe delete error, but proceeding") } if err := gn.RunInTransaction(func(gn *goon.Goon) error { if err := gn.Get(&u); err != nil && err != datastore.ErrNoSuchEntity { return err } u.Account = AFree u.Until = uc.Next if err := gn.Delete(gn.Key(&uc)); err != nil { return err } _, err := gn.Put(&u) return err }, nil); err != nil { return nil, err } return &uc, nil }
func UploadOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) { opml := Opml{} if err := json.Unmarshal([]byte(r.FormValue("opml")), &opml.Outline); err != nil { serveError(w, err) return } backupOPML(c) cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} ud := UserData{Id: "data", Parent: gn.Key(&u)} if err := gn.Get(&ud); err != nil { serveError(w, err) c.Errorf("get err: %v", err) return } if b, err := json.Marshal(&opml); err != nil { serveError(w, err) c.Errorf("json err: %v", err) return } else { l := Log{ Parent: ud.Parent, Id: time.Now().UnixNano(), Text: fmt.Sprintf("upload opml: %v -> %v", len(ud.Opml), len(b)), } ud.Opml = b if _, err := gn.PutMany(&ud, &l); err != nil { serveError(w, err) return } backupOPML(c) } }
func FeedHistory(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} uk := gn.Key(&u) if v := r.FormValue("v"); len(v) == 0 { q := datastore.NewQuery(gn.Key(&UserOpml{}).Kind()).Ancestor(uk).KeysOnly() keys, err := gn.GetAll(q, nil) if err != nil { serveError(w, err) return } times := make([]string, len(keys)) for i, k := range keys { times[i] = strconv.FormatInt(k.IntID(), 10) } b, _ := json.Marshal(×) w.Write(b) } else { a, _ := strconv.ParseInt(v, 10, 64) uo := UserOpml{Id: a, Parent: uk} c.Errorf("k: %v", uo) if err := gn.Get(&uo); err != nil { serveError(w, err) return } downloadOpml(w, uo.opml(), cu.Email) } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now()) q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron it := q.Run(appengine.Timeout(c, time.Minute)) tc := make(chan *taskqueue.Task) done := make(chan bool) i := 0 u := routeUrl("update-feed") go taskSender(c, "update-feed", tc, done) for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } tc <- taskqueue.NewPOSTTask(u, url.Values{ "feed": {k.StringID()}, }) i++ } close(tc) <-done c.Infof("updating %d feeds", i) }
func Donate(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} if err := gn.Get(&u); err != nil { serveError(w, err) return } amount, err := strconv.Atoi(r.FormValue("amount")) if err != nil || amount < 200 { serveError(w, fmt.Errorf("bad amount: %v", r.FormValue("amount"))) return } resp, err := stripe(c, "POST", "charges", url.Values{ "amount": {r.FormValue("amount")}, "description": {fmt.Sprintf("%v - %v", u.Id, u.Email)}, "card": {r.FormValue("stripeToken")}, "currency": {"usd"}, }.Encode()) if err != nil { serveError(w, err) return } else if resp.StatusCode != http.StatusOK { c.Errorf("%s", resp.Body) serveError(w, fmt.Errorf("Error")) return } }
func UploadOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) { opml := Opml{} if err := json.Unmarshal([]byte(r.FormValue("opml")), &opml.Outline); err != nil { serveError(w, err) return } backupOPML(c) cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} ud := UserData{Id: "data", Parent: gn.Key(&u)} if err := gn.Get(&ud); err != nil { serveError(w, err) c.Errorf("get err: %v", err) return } if b, err := json.Marshal(&opml); err != nil { saveError(c, fmt.Sprintf("%v", opml), err) serveError(w, err) c.Errorf("json err: %v", err) return } else { ud.Opml = b } gn.Put(&ud) backupOPML(c) }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()).Limit(3000) it := gn.Run(q) i := 0 for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } i++ } c.Infof("updating %d feeds", i) fmt.Fprintf(w, "updating %d feeds", i) }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) vars := mux.Vars(r) b, _ := base64.URLEncoding.DecodeString(vars["feed"]) f := Feed{Url: string(b)} if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.Put(&f) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else { c.Infof("push: %v", f.Url) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss := ParseFeed(c, f.Url, b) err := updateFeed(c, f.Url, nf, ss) if err != nil { c.Errorf("push error: %v", err) } } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) retry, _ := strconv.Atoi(r.FormValue("retry")) c.Errorf("retry: %v", retry) //* iterator it := gn.Run(q) i := 0 done := false for { k, err := it.Next(nil) if err == datastore.Done { done = true break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } i++ } c.Infof("updating %d feeds", i) fmt.Fprintf(w, "updating %d feeds", i) if !done { time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{ "retry": {strconv.Itoa(retry + 1)}, }) if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil { c.Errorf("taskqueue update feeds error: %v", err.Error()) } c.Errorf("ran update again") fmt.Fprintf(w, "\nran update again") } //*/ /* get all q = q.Limit(1000) keys, _ := gn.GetAll(q, nil) for _, k := range keys { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) fmt.Fprintf(w, "updating %d feeds", len(keys)) //*/ }
func ImportOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} if err := gn.Get(&u); err != nil { serveError(w, err) return } backupOPML(c) if file, _, err := r.FormFile("file"); err == nil { if fdata, err := ioutil.ReadAll(file); err == nil { buf := bytes.NewReader(fdata) // attempt to extract from google reader takeout zip if zb, zerr := zip.NewReader(buf, int64(len(fdata))); zerr == nil { for _, f := range zb.File { if strings.HasSuffix(f.FileHeader.Name, "Reader/subscriptions.xml") { if rc, rerr := f.Open(); rerr == nil { if fb, ferr := ioutil.ReadAll(rc); ferr == nil { fdata = fb break } } } } } // Preflight the OPML, so we can report any errors. d := xml.NewDecoder(bytes.NewReader(fdata)) d.CharsetReader = charset.NewReader d.Strict = false opml := Opml{} if err := d.Decode(&opml); err != nil { serveError(w, err) c.Errorf("opml error: %v", err.Error()) return } var b bytes.Buffer enc := gob.NewEncoder(&b) err := enc.Encode(&opml) if err != nil { serveError(w, err) return } bk, err := saveFile(c, b.Bytes()) if err != nil { serveError(w, err) return } task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{ "key": {string(bk)}, "user": {cu.ID}, }) taskqueue.Add(c, task, "import-reader") } } }
func SitemapFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) feed := vars["feed"] fk, err := datastore.DecodeKey(feed) if err != nil { serveError(w, err) return } bf := base64.URLEncoding.EncodeToString([]byte(fk.StringID())) q := datastore.NewQuery("S").KeysOnly().Ancestor(fk) q = q.Limit(Limit) cs := r.FormValue("c") if len(cs) > 0 { if cur, err := datastore.DecodeCursor(cs); err == nil { q = q.Start(cur) } } stories := make(map[string]string) it := q.Run(c) for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err) break } stories[k.StringID()] = base64.URLEncoding.EncodeToString([]byte(k.StringID())) } cs = "" if len(stories) == Limit { if cur, err := it.Cursor(); err == nil { cs = cur.String() } } if err := templates.ExecuteTemplate(w, "sitemap-feed.html", struct { Feed, Feed64 string Stories map[string]string Cursor string }{ Feed: feed, Feed64: bf, Stories: stories, Cursor: cs, }); err != nil { c.Errorf("%v", err) serveError(w, err) return } }
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) f := Feed{Url: r.FormValue("feed")} if err := gn.Get(&f); err != nil { c.Errorf("%v: %v", err, f.Url) serveError(w, err) return } else if f.IsSubscribed() { return } u := url.Values{} u.Add("hub.callback", f.PubSubURL()) u.Add("hub.mode", "subscribe") u.Add("hub.verify", "sync") fu, _ := url.Parse(f.Url) fu.Fragment = "" u.Add("hub.topic", fu.String()) req, err := http.NewRequest("POST", PUBSUBHUBBUB_HUB, strings.NewReader(u.Encode())) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") cl := urlfetch.Client(c) resp, err := cl.Do(req) if err != nil { c.Errorf("req error: %v", err) } else if resp.StatusCode != 204 { c.Errorf("resp: %v - %v", f.Url, resp.Status) c.Errorf("%s", resp.Body) } else { c.Infof("subscribed: %v", f.Url) } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) keys, _ := gn.GetAll(q, nil) for _, k := range keys { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) }
func Charge(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} uc := &UserCharge{Id: 1, Parent: gn.Key(&u)} if err := gn.Get(&u); err != nil { serveError(w, err) return } else if u.Account != AFree { serveError(w, fmt.Errorf("You're already subscribed.")) return } if err := gn.Get(uc); err == nil && len(uc.Customer) > 0 { serveError(w, fmt.Errorf("You're already subscribed.")) return } else if err != datastore.ErrNoSuchEntity { serveError(w, err) return } resp, err := stripe(c, "POST", "customers", url.Values{ "email": {u.Email}, "description": {u.Id}, "card": {r.FormValue("token")}, "plan": {r.FormValue("plan")}, }.Encode()) if err != nil { serveError(w, err) return } else if resp.StatusCode != http.StatusOK { var se StripeError defer resp.Body.Close() b, _ := ioutil.ReadAll(resp.Body) if err := json.Unmarshal(b, &se); err == nil { serveError(w, fmt.Errorf(se.Error.Message)) } else { serveError(w, fmt.Errorf("Error")) } c.Errorf("status: %v, %s", resp.StatusCode, b) return } uc, err = setCharge(c, resp) if err != nil { serveError(w, err) return } b, _ := json.Marshal(&uc) w.Write(b) }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) furl := r.FormValue("feed") oldURL := false if len(furl) == 0 { vars := mux.Vars(r) furl = vars["feed"] oldURL = true } b, _ := base64.URLEncoding.DecodeString(furl) f := Feed{Url: string(b)} c.Infof("url: %v", f.Url) if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if oldURL { c.Warningf("old url") http.Error(w, "", http.StatusNotFound) return } if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.Put(&f) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else if !f.NotViewed() { c.Infof("push: %v", f.Url) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss, err := ParseFeed(c, f.Url, b) if err != nil { c.Errorf("parse error: %v", err) return } if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil { c.Errorf("push error: %v", err) } } else { c.Infof("not viewed") } }
func DeleteAccount(c mpg.Context, w http.ResponseWriter, r *http.Request) { if _, err := doUncheckout(c); err != nil { c.Errorf("uncheckout err: %v", err) } cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} ud := UserData{Id: "data", Parent: gn.Key(&u)} if err := gn.Get(&u); err != nil { serveError(w, err) return } gn.Delete(gn.Key(&ud)) gn.Delete(ud.Parent) http.Redirect(w, r, routeUrl("logout"), http.StatusFound) }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) q = q.Limit(1000) it := gn.Run(q) var keys []*datastore.Key var del []*datastore.Key for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } else if len(k.StringID()) == 0 { del = append(del, k) continue } keys = append(keys, k) } tasks := make([]*taskqueue.Task, len(keys)) for i, k := range keys { tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) if len(del) > 0 { c.Errorf("attempt to delete %v feeds", len(del)) if err := gn.DeleteMulti(del); err != nil { c.Errorf("delete error: %v", err.Error()) } fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`) fmt.Fprintf(w, "attempt to delete %v feeds", len(del)) for _, k := range del { fmt.Fprintf(w, "\n<br>%v", k) } } fmt.Fprintf(w, "updating %d feeds", len(keys)) }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) furl := r.FormValue("feed") b, _ := base64.URLEncoding.DecodeString(furl) f := Feed{Url: string(b)} c.Infof("url: %v", f.Url) if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.PutMany(&f, &Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "SubscribeCallback - subscribed - " + f.Subscribed.String(), }) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else if !f.NotViewed() { c.Infof("push: %v", f.Url) gn.Put(&Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "SubscribeCallback - push update", }) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss, err := ParseFeed(c, f.Url, f.Url, b) if err != nil { c.Errorf("parse error: %v", err) return } if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil { c.Errorf("push error: %v", err) } } else { c.Infof("not viewed") } }
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed %s", url) f := Feed{Url: url} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { c.Errorf("no such entity") return } else if err != nil { c.Errorf("badurl7 error: %v", err.Error()) return } else if time.Now().Before(f.NextUpdate) { c.Infof("feed %v already updated", url) return } if f.Url == "" { c.Criticalf("badurl7: %v", url) return } feedError := func() { f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate) } c.Infof("fetching") if feed, stories := fetchFeed(c, f.Url, f.Url); feed != nil { if err := updateFeed(c, f.Url, feed, stories); err != nil { feedError() } } else { feedError() } c.Infof("done") }
// Task used to subscribe a feed to push. func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { start := time.Now() gn := goon.FromContext(c) f := Feed{Url: r.FormValue("feed")} fk := gn.Key(&f) s := "" defer func() { gn.Put(&Log{ Parent: fk, Id: time.Now().UnixNano(), Text: "SubscribeFeed - start " + start.String() + " - f.sub " + f.Subscribed.String() + " - " + s, }) }() if err := gn.Get(&f); err != nil { c.Errorf("%v: %v", err, f.Url) serveError(w, err) s += "err" return } else if f.IsSubscribed() { s += "is subscribed" return } u := url.Values{} u.Add("hub.callback", f.PubSubURL()) u.Add("hub.mode", "subscribe") u.Add("hub.verify", "sync") fu, _ := url.Parse(f.Url) fu.Fragment = "" u.Add("hub.topic", fu.String()) req, err := http.NewRequest("POST", f.Hub, strings.NewReader(u.Encode())) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") cl := &http.Client{ Transport: &urlfetch.Transport{ Context: c, Deadline: time.Minute, }, } resp, err := cl.Do(req) if err != nil { c.Errorf("req error: %v", err) } else if resp.StatusCode != http.StatusNoContent { f.Subscribed = time.Now().Add(time.Hour * 48) gn.Put(&f) if resp.StatusCode != http.StatusConflict { c.Errorf("resp: %v - %v", f.Url, resp.Status) c.Errorf("%s", resp.Body) } s += "resp err" } else { c.Infof("subscribed: %v", f.Url) s += "success" } }
func ClearRead(c mpg.Context, w http.ResponseWriter, r *http.Request) { if !isDevServer { return } cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} if err := gn.Get(u); err != nil { c.Errorf("err: %v", err.Error()) return } gn.Get(ud) u.Read = time.Time{} ud.Read = nil gn.PutMulti([]interface{}{u, ud}) http.Redirect(w, r, "/", http.StatusFound) }
func SetStar(c mpg.Context, w http.ResponseWriter, r *http.Request) { feed := r.FormValue("feed") story := r.FormValue("story") if len(feed) == 0 || len(story) == 0 { return } del := r.FormValue("del") != "" us := starKey(c, feed, story) gn := goon.FromContext(c) if del { gn.Delete(gn.Key(us)) } else { us.Created = time.Now() _, err := gn.Put(us) if err != nil { c.Errorf("star put err: %v", err) serveError(w, err) } } }
func AddSubscription(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) url := r.FormValue("url") o := &OpmlOutline{ Outline: []*OpmlOutline{ &OpmlOutline{XmlUrl: url}, }, } if err := addFeed(c, cu.ID, o); err != nil { c.Errorf("add sub error (%s): %s", url, err.Error()) serveError(w, err) return } gn := goon.FromContext(c) ud := UserData{Id: "data", Parent: gn.Key(&User{Id: cu.ID})} gn.Get(&ud) mergeUserOpml(&ud, o) gn.Put(&ud) }
func backupOPML(c mpg.Context) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} ud := UserData{Id: "data", Parent: gn.Key(&u)} if err := gn.Get(&ud); err != nil { return } uo := UserOpml{Id: time.Now().UnixNano(), Parent: gn.Key(&u)} buf := &bytes.Buffer{} if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil { gz.Write([]byte(ud.Opml)) gz.Close() uo.Compressed = buf.Bytes() } else { c.Errorf("gz err: %v", err) uo.Opml = ud.Opml } gn.Put(&uo) }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) q = q.Limit(2500) it := gn.Run(q) var keys []*datastore.Key for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } keys = append(keys, k) } tasks := make([]*taskqueue.Task, len(keys)) for i, k := range keys { tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) fmt.Fprintf(w, "updating %d feeds", len(keys)) }
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed %s", url) last := len(r.FormValue("last")) > 0 f := Feed{Url: url} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { c.Errorf("no such entity") return } else if err != nil { return } else if last { // noop } else if time.Now().Before(f.NextUpdate) { c.Infof("feed %v already updated: %v", url, f.NextUpdate) return } f.Subscribe(c) feedError := func(err error) { f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err) } if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil { if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil { feedError(err) } } else { feedError(err) } }