func Main(c mpg.Context, w http.ResponseWriter, r *http.Request) {
if err := templates.ExecuteTemplate(w, "base.html", includes(c, w, r)); err != nil {
c.Errorf("%v", err)
serveError(w, err)
return
}
}
func CFix(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
url := r.FormValue("feed")
c.Infof("fix feed %s", url)
f := Feed{Url: url}
if err := gn.Get(&f); err != nil {
c.Criticalf("cfix err: %v", err)
serveError(w, err)
return
}
q := datastore.NewQuery("S").Ancestor(gn.Key(&f))
var ss []*Story
keys, err := q.GetAll(c, &ss)
if err != nil {
c.Errorf("getall err: %v", err)
serveError(w, err)
return
}
c.Infof("trying to fix %v stories", len(ss))
const putLimit = 500
for i := 0; i <= len(keys)/putLimit; i++ {
lo := i * putLimit
hi := (i + 1) * putLimit
if hi > len(keys) {
hi = len(keys)
}
c.Infof("%v - %v", lo, hi)
if _, err := datastore.PutMulti(c, keys[lo:hi], ss[lo:hi]); err != nil {
c.Errorf("err: %v, %v, %v", lo, hi, err)
}
}
}
func AddSubscription(c mpg.Context, w http.ResponseWriter, r *http.Request) {
backupOPML(c)
cu := user.Current(c)
url := r.FormValue("url")
o := &OpmlOutline{
Outline: []*OpmlOutline{
&OpmlOutline{XmlUrl: url},
},
}
if err := addFeed(c, cu.ID, o); err != nil {
c.Errorf("add sub error (%s): %s", url, err.Error())
serveError(w, err)
return
}
gn := goon.FromContext(c)
ud := UserData{Id: "data", Parent: gn.Key(&User{Id: cu.ID})}
gn.Get(&ud)
if err := mergeUserOpml(c, &ud, o); err != nil {
c.Errorf("add sub error opml (%v): %v", url, err)
serveError(w, err)
return
}
gn.PutMany(&ud, &Log{
Parent: ud.Parent,
Id: time.Now().UnixNano(),
Text: fmt.Sprintf("add sub: %v", url),
})
if r.Method == "GET" {
http.Redirect(w, r, routeUrl("main"), http.StatusFound)
}
backupOPML(c)
}
func Charge(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
uc := UserCharge{Id: 1, Parent: gn.Key(&u)}
if err := gn.Get(&u); err != nil {
serveError(w, err)
return
} else if u.Account != AFree {
serveError(w, fmt.Errorf("You're already subscribed."))
return
}
if err := gn.Get(&uc); err == nil && len(uc.Customer) > 0 {
serveError(w, fmt.Errorf("You're already subscribed."))
return
} else if err != datastore.ErrNoSuchEntity {
serveError(w, err)
return
}
resp, err := stripe(c, "POST", "customers", url.Values{
"email": {u.Email},
"description": {u.Id},
"card": {r.FormValue("stripeToken")},
"plan": {STRIPE_PLAN},
}.Encode())
if err != nil {
serveError(w, err)
return
} else if resp.StatusCode != http.StatusOK {
c.Errorf("%s", resp.Body)
serveError(w, fmt.Errorf("Error"))
return
}
defer resp.Body.Close()
b, _ := ioutil.ReadAll(resp.Body)
var sc StripeCustomer
if err := json.Unmarshal(b, &sc); err != nil {
serveError(w, err)
return
}
if err := gn.RunInTransaction(func(gn *goon.Goon) error {
if err := gn.Get(&u); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
if err := gn.Get(&uc); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
u.Account = APaid
uc.Customer = sc.Id
uc.Last4 = sc.Card.Last4
uc.Created = time.Unix(sc.Created, 0)
_, err := gn.PutMany(&u, &uc)
return err
}, nil); err != nil {
serveError(w, err)
return
}
b, _ = json.Marshal(&uc)
w.Write(b)
}
func doUncheckout(c mpg.Context) (*UserCharge, error) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
uc := UserCharge{Id: 1, Parent: gn.Key(&u)}
if err := gn.Get(&u); err != nil {
return nil, err
}
if err := gn.Get(&uc); err != nil || len(uc.Customer) == 0 {
return nil, err
}
resp, err := stripe(c, "DELETE", "customers/"+uc.Customer, "")
if err != nil {
return nil, err
} else if resp.StatusCode != http.StatusOK {
c.Errorf("%s", resp.Body)
c.Errorf("stripe delete error, but proceeding")
}
if err := gn.RunInTransaction(func(gn *goon.Goon) error {
if err := gn.Get(&u); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
u.Account = AFree
u.Until = uc.Next
if err := gn.Delete(gn.Key(&uc)); err != nil {
return err
}
_, err := gn.Put(&u)
return err
}, nil); err != nil {
return nil, err
}
return &uc, nil
}
func UploadOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
opml := Opml{}
if err := json.Unmarshal([]byte(r.FormValue("opml")), &opml.Outline); err != nil {
serveError(w, err)
return
}
backupOPML(c)
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
ud := UserData{Id: "data", Parent: gn.Key(&u)}
if err := gn.Get(&ud); err != nil {
serveError(w, err)
c.Errorf("get err: %v", err)
return
}
if b, err := json.Marshal(&opml); err != nil {
serveError(w, err)
c.Errorf("json err: %v", err)
return
} else {
l := Log{
Parent: ud.Parent,
Id: time.Now().UnixNano(),
Text: fmt.Sprintf("upload opml: %v -> %v", len(ud.Opml), len(b)),
}
ud.Opml = b
if _, err := gn.PutMany(&ud, &l); err != nil {
serveError(w, err)
return
}
backupOPML(c)
}
}
func FeedHistory(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
uk := gn.Key(&u)
if v := r.FormValue("v"); len(v) == 0 {
q := datastore.NewQuery(gn.Key(&UserOpml{}).Kind()).Ancestor(uk).KeysOnly()
keys, err := gn.GetAll(q, nil)
if err != nil {
serveError(w, err)
return
}
times := make([]string, len(keys))
for i, k := range keys {
times[i] = strconv.FormatInt(k.IntID(), 10)
}
b, _ := json.Marshal(×)
w.Write(b)
} else {
a, _ := strconv.ParseInt(v, 10, 64)
uo := UserOpml{Id: a, Parent: uk}
c.Errorf("k: %v", uo)
if err := gn.Get(&uo); err != nil {
serveError(w, err)
return
}
downloadOpml(w, uo.opml(), cu.Email)
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
it := q.Run(appengine.Timeout(c, time.Minute))
tc := make(chan *taskqueue.Task)
done := make(chan bool)
i := 0
u := routeUrl("update-feed")
go taskSender(c, "update-feed", tc, done)
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
tc <- taskqueue.NewPOSTTask(u, url.Values{
"feed": {k.StringID()},
})
i++
}
close(tc)
<-done
c.Infof("updating %d feeds", i)
}
func Donate(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
if err := gn.Get(&u); err != nil {
serveError(w, err)
return
}
amount, err := strconv.Atoi(r.FormValue("amount"))
if err != nil || amount < 200 {
serveError(w, fmt.Errorf("bad amount: %v", r.FormValue("amount")))
return
}
resp, err := stripe(c, "POST", "charges", url.Values{
"amount": {r.FormValue("amount")},
"description": {fmt.Sprintf("%v - %v", u.Id, u.Email)},
"card": {r.FormValue("stripeToken")},
"currency": {"usd"},
}.Encode())
if err != nil {
serveError(w, err)
return
} else if resp.StatusCode != http.StatusOK {
c.Errorf("%s", resp.Body)
serveError(w, fmt.Errorf("Error"))
return
}
}
func UploadOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
opml := Opml{}
if err := json.Unmarshal([]byte(r.FormValue("opml")), &opml.Outline); err != nil {
serveError(w, err)
return
}
backupOPML(c)
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
ud := UserData{Id: "data", Parent: gn.Key(&u)}
if err := gn.Get(&ud); err != nil {
serveError(w, err)
c.Errorf("get err: %v", err)
return
}
if b, err := json.Marshal(&opml); err != nil {
saveError(c, fmt.Sprintf("%v", opml), err)
serveError(w, err)
c.Errorf("json err: %v", err)
return
} else {
ud.Opml = b
}
gn.Put(&ud)
backupOPML(c)
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now()).Limit(3000)
it := gn.Run(q)
i := 0
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
i++
}
c.Infof("updating %d feeds", i)
fmt.Fprintf(w, "updating %d feeds", i)
}
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
vars := mux.Vars(r)
b, _ := base64.URLEncoding.DecodeString(vars["feed"])
f := Feed{Url: string(b)}
if err := gn.Get(&f); err != nil {
http.Error(w, "", http.StatusNotFound)
return
}
if r.Method == "GET" {
if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
http.Error(w, "", http.StatusNotFound)
return
}
w.Write([]byte(r.FormValue("hub.challenge")))
i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
gn.Put(&f)
c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
return
} else {
c.Infof("push: %v", f.Url)
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
nf, ss := ParseFeed(c, f.Url, b)
err := updateFeed(c, f.Url, nf, ss)
if err != nil {
c.Errorf("push error: %v", err)
}
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
retry, _ := strconv.Atoi(r.FormValue("retry"))
c.Errorf("retry: %v", retry)
//* iterator
it := gn.Run(q)
i := 0
done := false
for {
k, err := it.Next(nil)
if err == datastore.Done {
done = true
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
i++
}
c.Infof("updating %d feeds", i)
fmt.Fprintf(w, "updating %d feeds", i)
if !done {
time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them
t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{
"retry": {strconv.Itoa(retry + 1)},
})
if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil {
c.Errorf("taskqueue update feeds error: %v", err.Error())
}
c.Errorf("ran update again")
fmt.Fprintf(w, "\nran update again")
}
//*/
/* get all
q = q.Limit(1000)
keys, _ := gn.GetAll(q, nil)
for _, k := range keys {
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
fmt.Fprintf(w, "updating %d feeds", len(keys))
//*/
}
func ImportOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
if err := gn.Get(&u); err != nil {
serveError(w, err)
return
}
backupOPML(c)
if file, _, err := r.FormFile("file"); err == nil {
if fdata, err := ioutil.ReadAll(file); err == nil {
buf := bytes.NewReader(fdata)
// attempt to extract from google reader takeout zip
if zb, zerr := zip.NewReader(buf, int64(len(fdata))); zerr == nil {
for _, f := range zb.File {
if strings.HasSuffix(f.FileHeader.Name, "Reader/subscriptions.xml") {
if rc, rerr := f.Open(); rerr == nil {
if fb, ferr := ioutil.ReadAll(rc); ferr == nil {
fdata = fb
break
}
}
}
}
}
// Preflight the OPML, so we can report any errors.
d := xml.NewDecoder(bytes.NewReader(fdata))
d.CharsetReader = charset.NewReader
d.Strict = false
opml := Opml{}
if err := d.Decode(&opml); err != nil {
serveError(w, err)
c.Errorf("opml error: %v", err.Error())
return
}
var b bytes.Buffer
enc := gob.NewEncoder(&b)
err := enc.Encode(&opml)
if err != nil {
serveError(w, err)
return
}
bk, err := saveFile(c, b.Bytes())
if err != nil {
serveError(w, err)
return
}
task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
"key": {string(bk)},
"user": {cu.ID},
})
taskqueue.Add(c, task, "import-reader")
}
}
}
func SitemapFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
feed := vars["feed"]
fk, err := datastore.DecodeKey(feed)
if err != nil {
serveError(w, err)
return
}
bf := base64.URLEncoding.EncodeToString([]byte(fk.StringID()))
q := datastore.NewQuery("S").KeysOnly().Ancestor(fk)
q = q.Limit(Limit)
cs := r.FormValue("c")
if len(cs) > 0 {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
}
}
stories := make(map[string]string)
it := q.Run(c)
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err)
break
}
stories[k.StringID()] = base64.URLEncoding.EncodeToString([]byte(k.StringID()))
}
cs = ""
if len(stories) == Limit {
if cur, err := it.Cursor(); err == nil {
cs = cur.String()
}
}
if err := templates.ExecuteTemplate(w, "sitemap-feed.html", struct {
Feed, Feed64 string
Stories map[string]string
Cursor string
}{
Feed: feed,
Feed64: bf,
Stories: stories,
Cursor: cs,
}); err != nil {
c.Errorf("%v", err)
serveError(w, err)
return
}
}
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
f := Feed{Url: r.FormValue("feed")}
if err := gn.Get(&f); err != nil {
c.Errorf("%v: %v", err, f.Url)
serveError(w, err)
return
} else if f.IsSubscribed() {
return
}
u := url.Values{}
u.Add("hub.callback", f.PubSubURL())
u.Add("hub.mode", "subscribe")
u.Add("hub.verify", "sync")
fu, _ := url.Parse(f.Url)
fu.Fragment = ""
u.Add("hub.topic", fu.String())
req, err := http.NewRequest("POST", PUBSUBHUBBUB_HUB, strings.NewReader(u.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
cl := urlfetch.Client(c)
resp, err := cl.Do(req)
if err != nil {
c.Errorf("req error: %v", err)
} else if resp.StatusCode != 204 {
c.Errorf("resp: %v - %v", f.Url, resp.Status)
c.Errorf("%s", resp.Body)
} else {
c.Infof("subscribed: %v", f.Url)
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
keys, _ := gn.GetAll(q, nil)
for _, k := range keys {
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
}
func Charge(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
uc := &UserCharge{Id: 1, Parent: gn.Key(&u)}
if err := gn.Get(&u); err != nil {
serveError(w, err)
return
} else if u.Account != AFree {
serveError(w, fmt.Errorf("You're already subscribed."))
return
}
if err := gn.Get(uc); err == nil && len(uc.Customer) > 0 {
serveError(w, fmt.Errorf("You're already subscribed."))
return
} else if err != datastore.ErrNoSuchEntity {
serveError(w, err)
return
}
resp, err := stripe(c, "POST", "customers", url.Values{
"email": {u.Email},
"description": {u.Id},
"card": {r.FormValue("token")},
"plan": {r.FormValue("plan")},
}.Encode())
if err != nil {
serveError(w, err)
return
} else if resp.StatusCode != http.StatusOK {
var se StripeError
defer resp.Body.Close()
b, _ := ioutil.ReadAll(resp.Body)
if err := json.Unmarshal(b, &se); err == nil {
serveError(w, fmt.Errorf(se.Error.Message))
} else {
serveError(w, fmt.Errorf("Error"))
}
c.Errorf("status: %v, %s", resp.StatusCode, b)
return
}
uc, err = setCharge(c, resp)
if err != nil {
serveError(w, err)
return
}
b, _ := json.Marshal(&uc)
w.Write(b)
}
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
furl := r.FormValue("feed")
oldURL := false
if len(furl) == 0 {
vars := mux.Vars(r)
furl = vars["feed"]
oldURL = true
}
b, _ := base64.URLEncoding.DecodeString(furl)
f := Feed{Url: string(b)}
c.Infof("url: %v", f.Url)
if err := gn.Get(&f); err != nil {
http.Error(w, "", http.StatusNotFound)
return
}
if r.Method == "GET" {
if oldURL {
c.Warningf("old url")
http.Error(w, "", http.StatusNotFound)
return
}
if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
http.Error(w, "", http.StatusNotFound)
return
}
w.Write([]byte(r.FormValue("hub.challenge")))
i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
gn.Put(&f)
c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
return
} else if !f.NotViewed() {
c.Infof("push: %v", f.Url)
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
nf, ss, err := ParseFeed(c, f.Url, b)
if err != nil {
c.Errorf("parse error: %v", err)
return
}
if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
c.Errorf("push error: %v", err)
}
} else {
c.Infof("not viewed")
}
}
func DeleteAccount(c mpg.Context, w http.ResponseWriter, r *http.Request) {
if _, err := doUncheckout(c); err != nil {
c.Errorf("uncheckout err: %v", err)
}
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
ud := UserData{Id: "data", Parent: gn.Key(&u)}
if err := gn.Get(&u); err != nil {
serveError(w, err)
return
}
gn.Delete(gn.Key(&ud))
gn.Delete(ud.Parent)
http.Redirect(w, r, routeUrl("logout"), http.StatusFound)
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(1000)
it := gn.Run(q)
var keys []*datastore.Key
var del []*datastore.Key
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
} else if len(k.StringID()) == 0 {
del = append(del, k)
continue
}
keys = append(keys, k)
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
if len(del) > 0 {
c.Errorf("attempt to delete %v feeds", len(del))
if err := gn.DeleteMulti(del); err != nil {
c.Errorf("delete error: %v", err.Error())
}
fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`)
fmt.Fprintf(w, "attempt to delete %v feeds", len(del))
for _, k := range del {
fmt.Fprintf(w, "\n<br>%v", k)
}
}
fmt.Fprintf(w, "updating %d feeds", len(keys))
}
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
furl := r.FormValue("feed")
b, _ := base64.URLEncoding.DecodeString(furl)
f := Feed{Url: string(b)}
c.Infof("url: %v", f.Url)
if err := gn.Get(&f); err != nil {
http.Error(w, "", http.StatusNotFound)
return
}
if r.Method == "GET" {
if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
http.Error(w, "", http.StatusNotFound)
return
}
w.Write([]byte(r.FormValue("hub.challenge")))
i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
gn.PutMany(&f, &Log{
Parent: gn.Key(&f),
Id: time.Now().UnixNano(),
Text: "SubscribeCallback - subscribed - " + f.Subscribed.String(),
})
c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
return
} else if !f.NotViewed() {
c.Infof("push: %v", f.Url)
gn.Put(&Log{
Parent: gn.Key(&f),
Id: time.Now().UnixNano(),
Text: "SubscribeCallback - push update",
})
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
nf, ss, err := ParseFeed(c, f.Url, f.Url, b)
if err != nil {
c.Errorf("parse error: %v", err)
return
}
if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
c.Errorf("push error: %v", err)
}
} else {
c.Infof("not viewed")
}
}
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
url := r.FormValue("feed")
c.Debugf("update feed %s", url)
f := Feed{Url: url}
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
c.Errorf("no such entity")
return
} else if err != nil {
c.Errorf("badurl7 error: %v", err.Error())
return
} else if time.Now().Before(f.NextUpdate) {
c.Infof("feed %v already updated", url)
return
}
if f.Url == "" {
c.Criticalf("badurl7: %v", url)
return
}
feedError := func() {
f.Errors++
v := f.Errors + 1
const max = 24 * 7
if v > max {
v = max
} else if f.Errors == 1 {
v = 0
}
f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
gn.Put(&f)
c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate)
}
c.Infof("fetching")
if feed, stories := fetchFeed(c, f.Url, f.Url); feed != nil {
if err := updateFeed(c, f.Url, feed, stories); err != nil {
feedError()
}
} else {
feedError()
}
c.Infof("done")
}
// Task used to subscribe a feed to push.
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
start := time.Now()
gn := goon.FromContext(c)
f := Feed{Url: r.FormValue("feed")}
fk := gn.Key(&f)
s := ""
defer func() {
gn.Put(&Log{
Parent: fk,
Id: time.Now().UnixNano(),
Text: "SubscribeFeed - start " + start.String() + " - f.sub " + f.Subscribed.String() + " - " + s,
})
}()
if err := gn.Get(&f); err != nil {
c.Errorf("%v: %v", err, f.Url)
serveError(w, err)
s += "err"
return
} else if f.IsSubscribed() {
s += "is subscribed"
return
}
u := url.Values{}
u.Add("hub.callback", f.PubSubURL())
u.Add("hub.mode", "subscribe")
u.Add("hub.verify", "sync")
fu, _ := url.Parse(f.Url)
fu.Fragment = ""
u.Add("hub.topic", fu.String())
req, err := http.NewRequest("POST", f.Hub, strings.NewReader(u.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
cl := &http.Client{
Transport: &urlfetch.Transport{
Context: c,
Deadline: time.Minute,
},
}
resp, err := cl.Do(req)
if err != nil {
c.Errorf("req error: %v", err)
} else if resp.StatusCode != http.StatusNoContent {
f.Subscribed = time.Now().Add(time.Hour * 48)
gn.Put(&f)
if resp.StatusCode != http.StatusConflict {
c.Errorf("resp: %v - %v", f.Url, resp.Status)
c.Errorf("%s", resp.Body)
}
s += "resp err"
} else {
c.Infof("subscribed: %v", f.Url)
s += "success"
}
}
func ClearRead(c mpg.Context, w http.ResponseWriter, r *http.Request) {
if !isDevServer {
return
}
cu := user.Current(c)
gn := goon.FromContext(c)
u := &User{Id: cu.ID}
ud := &UserData{Id: "data", Parent: gn.Key(u)}
if err := gn.Get(u); err != nil {
c.Errorf("err: %v", err.Error())
return
}
gn.Get(ud)
u.Read = time.Time{}
ud.Read = nil
gn.PutMulti([]interface{}{u, ud})
http.Redirect(w, r, "/", http.StatusFound)
}
func SetStar(c mpg.Context, w http.ResponseWriter, r *http.Request) {
feed := r.FormValue("feed")
story := r.FormValue("story")
if len(feed) == 0 || len(story) == 0 {
return
}
del := r.FormValue("del") != ""
us := starKey(c, feed, story)
gn := goon.FromContext(c)
if del {
gn.Delete(gn.Key(us))
} else {
us.Created = time.Now()
_, err := gn.Put(us)
if err != nil {
c.Errorf("star put err: %v", err)
serveError(w, err)
}
}
}
func AddSubscription(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
url := r.FormValue("url")
o := &OpmlOutline{
Outline: []*OpmlOutline{
&OpmlOutline{XmlUrl: url},
},
}
if err := addFeed(c, cu.ID, o); err != nil {
c.Errorf("add sub error (%s): %s", url, err.Error())
serveError(w, err)
return
}
gn := goon.FromContext(c)
ud := UserData{Id: "data", Parent: gn.Key(&User{Id: cu.ID})}
gn.Get(&ud)
mergeUserOpml(&ud, o)
gn.Put(&ud)
}
func backupOPML(c mpg.Context) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
ud := UserData{Id: "data", Parent: gn.Key(&u)}
if err := gn.Get(&ud); err != nil {
return
}
uo := UserOpml{Id: time.Now().UnixNano(), Parent: gn.Key(&u)}
buf := &bytes.Buffer{}
if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil {
gz.Write([]byte(ud.Opml))
gz.Close()
uo.Compressed = buf.Bytes()
} else {
c.Errorf("gz err: %v", err)
uo.Opml = ud.Opml
}
gn.Put(&uo)
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(2500)
it := gn.Run(q)
var keys []*datastore.Key
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
fmt.Fprintf(w, "updating %d feeds", len(keys))
}
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
url := r.FormValue("feed")
c.Debugf("update feed %s", url)
last := len(r.FormValue("last")) > 0
f := Feed{Url: url}
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
c.Errorf("no such entity")
return
} else if err != nil {
return
} else if last {
// noop
} else if time.Now().Before(f.NextUpdate) {
c.Infof("feed %v already updated: %v", url, f.NextUpdate)
return
}
f.Subscribe(c)
feedError := func(err error) {
f.Errors++
v := f.Errors + 1
const max = 24 * 7
if v > max {
v = max
} else if f.Errors == 1 {
v = 0
}
f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
gn.Put(&f)
c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
}
if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
feedError(err)
}
} else {
feedError(err)
}
}
