func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
it := q.Run(appengine.Timeout(c, time.Minute))
tc := make(chan *taskqueue.Task)
done := make(chan bool)
i := 0
u := routeUrl("update-feed")
go taskSender(c, "update-feed", tc, done)
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
tc <- taskqueue.NewPOSTTask(u, url.Values{
"feed": {k.StringID()},
})
i++
}
close(tc)
<-done
c.Infof("updating %d feeds", i)
}
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
url := r.FormValue("feed")
c.Debugf("update feed %s", url)
f := Feed{Url: url}
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
return
} else if time.Now().Before(f.NextUpdate) {
c.Infof("feed %v already updated", url)
return
}
if feed, stories := fetchFeed(c, url, url); feed != nil {
updateFeed(c, url, feed, stories)
} else {
f.Errors++
v := f.Errors + 1
const max = 24 * 7
if v > max {
v = max
} else if f.Errors == 1 {
v = 0
}
f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
gn.Put(&f)
c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate)
}
}
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
vars := mux.Vars(r)
b, _ := base64.URLEncoding.DecodeString(vars["feed"])
f := Feed{Url: string(b)}
if err := gn.Get(&f); err != nil {
http.Error(w, "", http.StatusNotFound)
return
}
if r.Method == "GET" {
if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
http.Error(w, "", http.StatusNotFound)
return
}
w.Write([]byte(r.FormValue("hub.challenge")))
i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
gn.Put(&f)
c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
return
} else {
c.Infof("push: %v", f.Url)
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
nf, ss := ParseFeed(c, f.Url, b)
err := updateFeed(c, f.Url, nf, ss)
if err != nil {
c.Errorf("push error: %v", err)
}
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now()).Limit(3000)
it := gn.Run(q)
i := 0
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
i++
}
c.Infof("updating %d feeds", i)
fmt.Fprintf(w, "updating %d feeds", i)
}
func DeleteOldFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
g := goon.FromContext(c)
oldDate := time.Now().Add(-time.Hour * 24 * 90)
feed := Feed{Url: r.FormValue("f")}
if err := g.Get(&feed); err != nil {
c.Criticalf("err: %v", err)
return
}
if feed.LastViewed.After(oldDate) {
return
}
q := datastore.NewQuery(g.Kind(&Story{})).Ancestor(g.Key(&feed)).KeysOnly()
keys, err := q.GetAll(c, nil)
if err != nil {
c.Criticalf("err: %v", err)
return
}
q = datastore.NewQuery(g.Kind(&StoryContent{})).Ancestor(g.Key(&feed)).KeysOnly()
sckeys, err := q.GetAll(c, nil)
if err != nil {
c.Criticalf("err: %v", err)
return
}
keys = append(keys, sckeys...)
c.Infof("delete: %v - %v", feed.Url, len(keys))
err = g.DeleteMulti(keys)
if err != nil {
c.Criticalf("err: %v", err)
}
}
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
f := Feed{Url: r.FormValue("feed")}
if err := gn.Get(&f); err != nil {
c.Errorf("%v: %v", err, f.Url)
serveError(w, err)
return
} else if f.IsSubscribed() {
return
}
u := url.Values{}
u.Add("hub.callback", f.PubSubURL())
u.Add("hub.mode", "subscribe")
u.Add("hub.verify", "sync")
fu, _ := url.Parse(f.Url)
fu.Fragment = ""
u.Add("hub.topic", fu.String())
req, err := http.NewRequest("POST", PUBSUBHUBBUB_HUB, strings.NewReader(u.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
cl := urlfetch.Client(c)
resp, err := cl.Do(req)
if err != nil {
c.Errorf("req error: %v", err)
} else if resp.StatusCode != 204 {
c.Errorf("resp: %v - %v", f.Url, resp.Status)
c.Errorf("%s", resp.Body)
} else {
c.Infof("subscribed: %v", f.Url)
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
retry, _ := strconv.Atoi(r.FormValue("retry"))
c.Errorf("retry: %v", retry)
//* iterator
it := gn.Run(q)
i := 0
done := false
for {
k, err := it.Next(nil)
if err == datastore.Done {
done = true
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
i++
}
c.Infof("updating %d feeds", i)
fmt.Fprintf(w, "updating %d feeds", i)
if !done {
time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them
t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{
"retry": {strconv.Itoa(retry + 1)},
})
if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil {
c.Errorf("taskqueue update feeds error: %v", err.Error())
}
c.Errorf("ran update again")
fmt.Fprintf(w, "\nran update again")
}
//*/
/* get all
q = q.Limit(1000)
keys, _ := gn.GetAll(q, nil)
for _, k := range keys {
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
fmt.Fprintf(w, "updating %d feeds", len(keys))
//*/
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(1000)
it := gn.Run(q)
var keys []*datastore.Key
var del []*datastore.Key
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
} else if len(k.StringID()) == 0 {
del = append(del, k)
continue
}
keys = append(keys, k)
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
if len(del) > 0 {
c.Errorf("attempt to delete %v feeds", len(del))
if err := gn.DeleteMulti(del); err != nil {
c.Errorf("delete error: %v", err.Error())
}
fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`)
fmt.Fprintf(w, "attempt to delete %v feeds", len(del))
for _, k := range del {
fmt.Fprintf(w, "\n<br>%v", k)
}
}
fmt.Fprintf(w, "updating %d feeds", len(keys))
}
func ClearFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
if !isDevServer {
return
}
cu := user.Current(c)
gn := goon.FromContext(c)
done := make(chan bool)
go func() {
u := &User{Id: cu.ID}
defer func() { done <- true }()
ud := &UserData{Id: "data", Parent: gn.Key(u)}
if err := gn.Get(u); err != nil {
c.Errorf("user del err: %v", err.Error())
return
}
gn.Get(ud)
u.Read = time.Time{}
ud.Read = nil
ud.Opml = nil
gn.PutMany(u, ud)
c.Infof("%v cleared", u.Email)
}()
del := func(kind string) {
defer func() { done <- true }()
q := datastore.NewQuery(kind).KeysOnly()
keys, err := gn.GetAll(q, nil)
if err != nil {
c.Errorf("err: %v", err.Error())
return
}
if err := gn.DeleteMulti(keys); err != nil {
c.Errorf("err: %v", err.Error())
return
}
c.Infof("%v deleted", kind)
}
types := []interface{}{
&Feed{},
&Story{},
&StoryContent{},
&DateFormat{},
&Log{},
&UserOpml{},
}
for _, i := range types {
k := gn.Key(i).Kind()
go del(k)
}
for i := 0; i < len(types); i++ {
<-done
}
http.Redirect(w, r, routeUrl("main"), http.StatusFound)
}
// Task used to subscribe a feed to push.
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
start := time.Now()
gn := goon.FromContext(c)
f := Feed{Url: r.FormValue("feed")}
fk := gn.Key(&f)
s := ""
defer func() {
gn.Put(&Log{
Parent: fk,
Id: time.Now().UnixNano(),
Text: "SubscribeFeed - start " + start.String() + " - f.sub " + f.Subscribed.String() + " - " + s,
})
}()
if err := gn.Get(&f); err != nil {
c.Errorf("%v: %v", err, f.Url)
serveError(w, err)
s += "err"
return
} else if f.IsSubscribed() {
s += "is subscribed"
return
}
u := url.Values{}
u.Add("hub.callback", f.PubSubURL())
u.Add("hub.mode", "subscribe")
u.Add("hub.verify", "sync")
fu, _ := url.Parse(f.Url)
fu.Fragment = ""
u.Add("hub.topic", fu.String())
req, err := http.NewRequest("POST", f.Hub, strings.NewReader(u.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
cl := &http.Client{
Transport: &urlfetch.Transport{
Context: c,
Deadline: time.Minute,
},
}
resp, err := cl.Do(req)
if err != nil {
c.Errorf("req error: %v", err)
} else if resp.StatusCode != http.StatusNoContent {
f.Subscribed = time.Now().Add(time.Hour * 48)
gn.Put(&f)
if resp.StatusCode != http.StatusConflict {
c.Errorf("resp: %v - %v", f.Url, resp.Status)
c.Errorf("%s", resp.Body)
}
s += "resp err"
} else {
c.Infof("subscribed: %v", f.Url)
s += "success"
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
keys, _ := gn.GetAll(q, nil)
for _, k := range keys {
t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(3000)
var keys []*datastore.Key
it := q.Run(c)
retry := 0
for {
k, err := it.Next(nil)
if err != nil {
c.Errorf("next error: %v, retry: %v", err.Error(), retry)
if retry == 5 {
break
}
retry++
it = q.Run(c)
} else {
keys = append(keys, k)
}
}
if len(keys) == 0 {
c.Errorf("giving up")
return
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
c.Infof("task: %v, %v", i, k)
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
}
func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error {
gn := goon.FromContext(c)
o := outline.Outline[0]
c.Infof("adding feed %v to user %s", o.XmlUrl, userid)
fu, ferr := url.Parse(o.XmlUrl)
if ferr != nil {
return ferr
}
fu.Fragment = ""
o.XmlUrl = fu.String()
f := Feed{Url: o.XmlUrl}
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
if feed, stories, err := fetchFeed(c, o.XmlUrl, o.XmlUrl); err != nil {
return fmt.Errorf("could not add feed %s: %v", o.XmlUrl, err)
} else {
f = *feed
f.Updated = time.Time{}
f.Checked = f.Updated
f.NextUpdate = f.Updated
f.LastViewed = time.Now()
gn.Put(&f)
for _, s := range stories {
s.Created = s.Published
}
if err := updateFeed(c, f.Url, feed, stories, false, false, false); err != nil {
return err
}
o.XmlUrl = feed.Url
o.HtmlUrl = feed.Link
if o.Title == "" {
o.Title = feed.Title
}
}
} else if err != nil {
return err
} else {
o.HtmlUrl = f.Link
if o.Title == "" {
o.Title = f.Title
}
}
o.Text = ""
return nil
}
func AdminUser(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Kind(&User{})).Limit(1)
q = q.Filter("e =", r.FormValue("u"))
it := gn.Run(q)
var u User
ud := UserData{Id: "data"}
var h []Log
k, err := it.Next(&u)
if err != nil {
serveError(w, err)
return
}
ud.Parent = gn.Key(&u)
gn.Get(&ud)
until := r.FormValue("until")
if d, err := time.Parse("2006-01-02", until); err == nil {
u.Until = d
gn.Put(&u)
}
if o := []byte(r.FormValue("opml")); len(o) > 0 {
opml := Opml{}
if err := json.Unmarshal(o, &opml); err != nil {
serveError(w, err)
return
}
ud.Opml = o
if _, err := gn.Put(&ud); err != nil {
serveError(w, err)
return
}
c.Infof("opml updated")
}
q = datastore.NewQuery(gn.Kind(&Log{})).Ancestor(k)
_, err = gn.GetAll(q, &h)
if err := templates.ExecuteTemplate(w, "admin-user.html", struct {
User User
Data UserData
Log []Log
}{
u,
ud,
h,
}); err != nil {
serveError(w, err)
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron
var keys []*datastore.Key
it := q.Run(appengine.Timeout(c, time.Second*60))
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
if len(keys) == 0 {
c.Errorf("no results")
return
}
c.Infof("updating %d feeds", len(keys))
var tasks []*taskqueue.Task
for _, k := range keys {
tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
}))
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
}
func taskSender(c mpg.Context, queue string, tc chan *taskqueue.Task, done chan bool) {
const taskLimit = 100
tasks := make([]*taskqueue.Task, 0, taskLimit)
send := func() {
taskqueue.AddMulti(c, tasks, queue)
c.Infof("added %v tasks", len(tasks))
tasks = tasks[0:0]
}
for t := range tc {
tasks = append(tasks, t)
if len(tasks) == taskLimit {
send()
}
}
if len(tasks) > 0 {
send()
}
done <- true
}
func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error {
gn := goon.FromContext(c)
o := outline.Outline[0]
c.Infof("adding feed %v to user %s", o.XmlUrl, userid)
f := Feed{Url: o.XmlUrl}
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
if feed, stories := fetchFeed(c, o.XmlUrl, o.XmlUrl); feed == nil {
return errors.New(fmt.Sprintf("could not add feed %s", o.XmlUrl))
} else {
f = *feed
f.Updated = time.Time{}
f.Checked = f.Updated
f.NextUpdate = f.Updated
if strings.TrimSpace(f.Url) == "" {
c.Criticalf("badurl4: %v, %v", o.XmlUrl, o)
return errors.New("badurl4")
}
gn.PutComplete(&f)
if err := updateFeed(c, f.Url, feed, stories); err != nil {
return err
}
o.XmlUrl = feed.Url
o.HtmlUrl = feed.Link
if o.Title == "" {
o.Title = feed.Title
}
}
} else if err != nil {
return err
} else {
o.HtmlUrl = f.Link
if o.Title == "" {
o.Title = f.Title
}
}
o.Text = ""
return nil
}
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
url := r.FormValue("feed")
c.Debugf("update feed %s", url)
last := len(r.FormValue("last")) > 0
f := Feed{Url: url}
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
c.Errorf("no such entity")
return
} else if err != nil {
return
} else if last {
// noop
} else if time.Now().Before(f.NextUpdate) {
c.Infof("feed %v already updated: %v", url, f.NextUpdate)
return
}
f.Subscribe(c)
feedError := func(err error) {
f.Errors++
v := f.Errors + 1
const max = 24 * 7
if v > max {
v = max
} else if f.Errors == 1 {
v = 0
}
f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
gn.Put(&f)
c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
}
if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
feedError(err)
}
} else {
feedError(err)
}
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(2500)
it := gn.Run(q)
var keys []*datastore.Key
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
fmt.Fprintf(w, "updating %d feeds", len(keys))
}
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(3000)
var keys []*datastore.Key
for i := 0; i < 5; i++ {
if _keys, err := q.GetAll(c, nil); err != nil {
c.Errorf("get all error: %v, retry %v", err.Error(), i)
} else {
c.Errorf("got %v keys", len(_keys))
keys = _keys
break
}
}
if len(keys) == 0 {
c.Errorf("giving up")
return
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
}
func Account(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := User{Id: cu.ID}
uc := &UserCharge{Id: 1, Parent: gn.Key(&u)}
if err := gn.Get(uc); err == nil {
if uc.Next.Before(time.Now()) {
if resp, err := stripe(c, "GET", "customers/"+uc.Customer, ""); err == nil {
if nuc, err := setCharge(c, resp); err == nil {
uc = nuc
c.Infof("updated user charge %v", cu.ID)
}
}
}
b, _ := json.Marshal(&uc)
w.Write(b)
} else if err != datastore.ErrNoSuchEntity {
serveError(w, err)
return
}
}
func CFix(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
url := r.FormValue("feed")
c.Infof("fix feed %s", url)
f := Feed{Url: url}
if err := gn.Get(&f); err != nil {
c.Criticalf("cfix err: %v", err)
serveError(w, err)
return
}
q := datastore.NewQuery("S").Ancestor(gn.Key(&f))
var ss []*Story
keys, err := q.GetAll(c, &ss)
if err != nil {
c.Errorf("getall err: %v", err)
serveError(w, err)
return
}
c.Infof("trying to fix %v stories", len(ss))
const putLimit = 500
for i := 0; i <= len(keys)/putLimit; i++ {
lo := i * putLimit
hi := (i + 1) * putLimit
if hi > len(keys) {
hi = len(keys)
}
c.Infof("%v - %v", lo, hi)
if _, err := datastore.PutMulti(c, keys[lo:hi], ss[lo:hi]); err != nil {
c.Errorf("err: %v, %v, %v", lo, hi, err)
}
}
}
func BackendStart(c mpg.Context, w http.ResponseWriter, r *http.Request) {
const sz = 100
ic := 0
gn := goon.FromContext(c)
fk := gn.Key(&Feed{Url: "a"})
q := datastore.NewQuery("F").Filter("__key__ <", fk).Order("__key__").KeysOnly().Limit(1)
keys, _ := q.GetAll(c, nil)
if len(keys) == 0 {
return
}
c.Errorf("start: %v", keys[0])
startid := keys[0].IntID() / sz
var f func(appengine.Context)
f = func(c appengine.Context) {
c.Errorf("new request: %d", ic)
t1 := time.Now()
wg := sync.WaitGroup{}
wg.Add(sz)
var j int64
for j = 0; j < sz; j++ {
go func(j int64) {
k := datastore.NewKey(c, "F", "", startid*sz+j, nil)
c.Infof("del: %v", k)
if err := datastore.Delete(c, k); err != nil {
c.Errorf("delete err: %v", err.Error())
}
wg.Done()
}(j)
}
wg.Wait()
t2 := time.Now()
c.Infof("%v, %v, %v", t1, t2, t2.Sub(t1))
ic++
startid++
runtime.RunInBackground(c, f)
}
runtime.RunInBackground(c, f)
}
func Search(c mp.Context, w http.ResponseWriter, r *http.Request) error {
query := r.URL.Query()
q := query.Get("q")
if q == "" {
return core.Render(w, "songs.search.tmpl.html", &SongSearchResult{Includes: core.NewIncludes(c)})
}
searchQuery := m.Wordify(q)
docList, docIDs, err := data.SearchSongs(c, searchQuery)
if err != nil {
return err
}
if len(docList) == 0 {
c.Infof("AND query [%s] return nothing", searchQuery)
orQuery := strings.Join(strings.Fields(searchQuery), " OR ")
c.Infof("Searching with OR query [%s]", orQuery)
docList, docIDs, err = data.SearchSongs(c, orQuery)
if err != nil {
return err
}
}
items := make([]*SongSearchResultItem, 0)
for i, _ := range docList {
items = append(items, createSongSearchResultItem(q, docIDs[i], docList[i]))
}
result := &SongSearchResult{
Includes: core.NewIncludes(c),
Query: q,
Items: items,
}
return core.Render(w, "songs.search.tmpl.html", result)
}
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
furl := r.FormValue("feed")
oldURL := false
if len(furl) == 0 {
vars := mux.Vars(r)
furl = vars["feed"]
oldURL = true
}
b, _ := base64.URLEncoding.DecodeString(furl)
f := Feed{Url: string(b)}
c.Infof("url: %v", f.Url)
if err := gn.Get(&f); err != nil {
http.Error(w, "", http.StatusNotFound)
return
}
if r.Method == "GET" {
if oldURL {
c.Warningf("old url")
http.Error(w, "", http.StatusNotFound)
return
}
if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
http.Error(w, "", http.StatusNotFound)
return
}
w.Write([]byte(r.FormValue("hub.challenge")))
i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
gn.Put(&f)
c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
return
} else if !f.NotViewed() {
c.Infof("push: %v", f.Url)
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
nf, ss, err := ParseFeed(c, f.Url, b)
if err != nil {
c.Errorf("parse error: %v", err)
return
}
if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
c.Errorf("push error: %v", err)
}
} else {
c.Infof("not viewed")
}
}
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
furl := r.FormValue("feed")
b, _ := base64.URLEncoding.DecodeString(furl)
f := Feed{Url: string(b)}
c.Infof("url: %v", f.Url)
if err := gn.Get(&f); err != nil {
http.Error(w, "", http.StatusNotFound)
return
}
if r.Method == "GET" {
if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
http.Error(w, "", http.StatusNotFound)
return
}
w.Write([]byte(r.FormValue("hub.challenge")))
i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
gn.PutMany(&f, &Log{
Parent: gn.Key(&f),
Id: time.Now().UnixNano(),
Text: "SubscribeCallback - subscribed - " + f.Subscribed.String(),
})
c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
return
} else if !f.NotViewed() {
c.Infof("push: %v", f.Url)
gn.Put(&Log{
Parent: gn.Key(&f),
Id: time.Now().UnixNano(),
Text: "SubscribeCallback - push update",
})
defer r.Body.Close()
b, _ := ioutil.ReadAll(r.Body)
nf, ss, err := ParseFeed(c, f.Url, f.Url, b)
if err != nil {
c.Errorf("parse error: %v", err)
return
}
if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
c.Errorf("push error: %v", err)
}
} else {
c.Infof("not viewed")
}
}
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error {
gn := goon.FromContext(c)
f := Feed{Url: url}
if err := gn.Get(&f); err != nil {
return errors.New(fmt.Sprintf("feed not found: %s", url))
}
// Compare the feed's listed update to the story's update.
// Note: these may not be accurate, hence, only compare them to each other,
// since they should have the same relative error.
storyDate := f.Updated
hasUpdated := !feed.Updated.IsZero()
isFeedUpdated := f.Updated == feed.Updated
if !hasUpdated {
feed.Updated = f.Updated
}
feed.Date = f.Date
f = *feed
if hasUpdated && isFeedUpdated {
c.Infof("feed %s already updated to %v, putting", url, feed.Updated)
f.Updated = time.Now()
if strings.TrimSpace(f.Url) == "" {
c.Criticalf("badurl5: %v, %v", url, f)
return errors.New("badurl5")
}
gn.PutComplete(&f)
return nil
}
c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate)
var newStories []*Story
for _, s := range stories {
if s.Updated.IsZero() || !s.Updated.Before(storyDate) {
newStories = append(newStories, s)
}
}
c.Debugf("%v possible stories to update", len(newStories))
puts := []interface{}{&f}
// find non existant stories
fk := gn.Key(&f)
getStories := make([]*Story, len(newStories))
for i, s := range newStories {
getStories[i] = &Story{Id: s.Id, Parent: fk}
}
err := gn.GetMulti(getStories)
if _, ok := err.(appengine.MultiError); err != nil && !ok {
c.Errorf("get multi error: %v", err.Error())
return err
}
var updateStories []*Story
for i, s := range getStories {
if goon.NotFound(err, i) {
updateStories = append(updateStories, newStories[i])
} else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) {
newStories[i].Created = s.Created
newStories[i].Published = s.Published
updateStories = append(updateStories, newStories[i])
}
}
c.Debugf("%v update stories", len(updateStories))
for _, s := range updateStories {
puts = append(puts, s)
gn.Put(&StoryContent{
Id: 1,
Parent: gn.Key(s),
Content: s.content,
})
}
c.Debugf("putting %v entities", len(puts))
if len(puts) > 1 {
f.Date = time.Now()
if !hasUpdated {
f.Updated = f.Date
}
}
if f.Url == "" {
c.Criticalf("badurl6: %v", f)
return errors.New("badurl6")
}
gn.PutMultiComplete(puts)
return nil
}
func CFixer(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Limit(1000)
cs := r.FormValue("c")
if len(cs) > 0 {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
c.Infof("starting at %v", cur)
} else {
c.Errorf("cursor error %v", err.Error())
}
}
var keys []*datastore.Key
it := q.Run(Timeout(c, time.Second*15))
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
if len(keys) == 0 {
c.Errorf("no results")
return
} else {
cur, err := it.Cursor()
if err != nil {
c.Errorf("to cur error %v", err.Error())
} else {
c.Infof("add with cur %v", cur)
t := taskqueue.NewPOSTTask("/tasks/cfixer", url.Values{
"c": {cur.String()},
})
taskqueue.Add(c, t, "cfixer")
}
}
c.Infof("fixing %d feeds", len(keys))
var tasks []*taskqueue.Task
for _, k := range keys {
c.Infof("f: %v", k.StringID())
tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/cfix", url.Values{
"feed": {k.StringID()},
}))
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "cfixer"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
}
func ImportOpmlTask(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
userid := r.FormValue("user")
bk := r.FormValue("key")
fr := blobstore.NewReader(c, appengine.BlobKey(bk))
var skip int
if s, err := strconv.Atoi(r.FormValue("skip")); err == nil {
skip = s
}
c.Debugf("reader import for %v, skip %v", userid, skip)
var userOpml []*OpmlOutline
remaining := skip
var proc func(label string, outlines []*OpmlOutline)
proc = func(label string, outlines []*OpmlOutline) {
for _, o := range outlines {
if o.Title == "" {
o.Title = o.Text
}
if o.XmlUrl != "" {
if remaining > 0 {
remaining--
} else if len(userOpml) < IMPORT_LIMIT {
userOpml = append(userOpml, &OpmlOutline{
Title: label,
Outline: []*OpmlOutline{o},
})
}
}
if o.Title != "" && len(o.Outline) > 0 {
proc(o.Title, o.Outline)
}
}
}
opml := Opml{}
d := xml.NewDecoder(fr)
d.CharsetReader = charset.NewReader
d.Strict = false
if err := d.Decode(&opml); err != nil {
c.Errorf("opml error: %v", err.Error())
return
}
proc("", opml.Outline)
// todo: refactor below with similar from ImportReaderTask
wg := sync.WaitGroup{}
wg.Add(len(userOpml))
for i := range userOpml {
go func(i int) {
o := userOpml[i].Outline[0]
if err := addFeed(c, userid, userOpml[i]); err != nil {
c.Warningf("opml import error: %v", err.Error())
// todo: do something here?
}
c.Debugf("opml import: %s, %s", o.Title, o.XmlUrl)
wg.Done()
}(i)
}
wg.Wait()
ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})}
if err := gn.RunInTransaction(func(gn *goon.Goon) error {
gn.Get(&ud)
mergeUserOpml(&ud, userOpml...)
_, err := gn.Put(&ud)
return err
}, nil); err != nil {
w.WriteHeader(http.StatusInternalServerError)
c.Errorf("ude update error: %v", err.Error())
return
}
if len(userOpml) == IMPORT_LIMIT {
task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
"key": {bk},
"user": {userid},
"skip": {strconv.Itoa(skip + IMPORT_LIMIT)},
})
taskqueue.Add(c, task, "import-reader")
} else {
c.Infof("opml import done: %v", userid)
}
}
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error {
gn := goon.FromContext(c)
f := Feed{Url: url}
if err := gn.Get(&f); err != nil {
return fmt.Errorf("feed not found: %s", url)
}
// Compare the feed's listed update to the story's update.
// Note: these may not be accurate, hence, only compare them to each other,
// since they should have the same relative error.
storyDate := f.Updated
hasUpdated := !feed.Updated.IsZero()
isFeedUpdated := f.Updated == feed.Updated
if !hasUpdated {
feed.Updated = f.Updated
}
feed.Date = f.Date
f = *feed
if hasUpdated && isFeedUpdated {
c.Infof("feed %s already updated to %v, putting", url, feed.Updated)
f.Updated = time.Now()
gn.Put(&f)
return nil
}
c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate)
var newStories []*Story
for _, s := range stories {
if s.Updated.IsZero() || !s.Updated.Before(storyDate) {
newStories = append(newStories, s)
}
}
c.Debugf("%v possible stories to update", len(newStories))
puts := []interface{}{&f}
// find non existant stories
fk := gn.Key(&f)
getStories := make([]*Story, len(newStories))
for i, s := range newStories {
getStories[i] = &Story{Id: s.Id, Parent: fk}
}
err := gn.GetMulti(getStories)
if _, ok := err.(appengine.MultiError); err != nil && !ok {
c.Errorf("get multi error: %v", err.Error())
return err
}
var updateStories []*Story
for i, s := range getStories {
if goon.NotFound(err, i) {
updateStories = append(updateStories, newStories[i])
} else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) {
newStories[i].Created = s.Created
newStories[i].Published = s.Published
updateStories = append(updateStories, newStories[i])
}
}
c.Debugf("%v update stories", len(updateStories))
for _, s := range updateStories {
puts = append(puts, s)
sc := StoryContent{
Id: 1,
Parent: gn.Key(s),
}
buf := &bytes.Buffer{}
if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil {
gz.Write([]byte(s.content))
gz.Close()
sc.Compressed = buf.Bytes()
}
if len(sc.Compressed) == 0 {
sc.Content = s.content
}
gn.Put(&sc)
}
c.Debugf("putting %v entities", len(puts))
if len(puts) > 1 {
f.Date = time.Now()
if !hasUpdated {
f.Updated = f.Date
}
}
gn.PutMulti(puts)
return nil
}