func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now()) q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron it := q.Run(appengine.Timeout(c, time.Minute)) tc := make(chan *taskqueue.Task) done := make(chan bool) i := 0 u := routeUrl("update-feed") go taskSender(c, "update-feed", tc, done) for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } tc <- taskqueue.NewPOSTTask(u, url.Values{ "feed": {k.StringID()}, }) i++ } close(tc) <-done c.Infof("updating %d feeds", i) }
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed %s", url) f := Feed{Url: url} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { return } else if time.Now().Before(f.NextUpdate) { c.Infof("feed %v already updated", url) return } if feed, stories := fetchFeed(c, url, url); feed != nil { updateFeed(c, url, feed, stories) } else { f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate) } }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) vars := mux.Vars(r) b, _ := base64.URLEncoding.DecodeString(vars["feed"]) f := Feed{Url: string(b)} if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.Put(&f) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else { c.Infof("push: %v", f.Url) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss := ParseFeed(c, f.Url, b) err := updateFeed(c, f.Url, nf, ss) if err != nil { c.Errorf("push error: %v", err) } } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()).Limit(3000) it := gn.Run(q) i := 0 for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } i++ } c.Infof("updating %d feeds", i) fmt.Fprintf(w, "updating %d feeds", i) }
func DeleteOldFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { g := goon.FromContext(c) oldDate := time.Now().Add(-time.Hour * 24 * 90) feed := Feed{Url: r.FormValue("f")} if err := g.Get(&feed); err != nil { c.Criticalf("err: %v", err) return } if feed.LastViewed.After(oldDate) { return } q := datastore.NewQuery(g.Kind(&Story{})).Ancestor(g.Key(&feed)).KeysOnly() keys, err := q.GetAll(c, nil) if err != nil { c.Criticalf("err: %v", err) return } q = datastore.NewQuery(g.Kind(&StoryContent{})).Ancestor(g.Key(&feed)).KeysOnly() sckeys, err := q.GetAll(c, nil) if err != nil { c.Criticalf("err: %v", err) return } keys = append(keys, sckeys...) c.Infof("delete: %v - %v", feed.Url, len(keys)) err = g.DeleteMulti(keys) if err != nil { c.Criticalf("err: %v", err) } }
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) f := Feed{Url: r.FormValue("feed")} if err := gn.Get(&f); err != nil { c.Errorf("%v: %v", err, f.Url) serveError(w, err) return } else if f.IsSubscribed() { return } u := url.Values{} u.Add("hub.callback", f.PubSubURL()) u.Add("hub.mode", "subscribe") u.Add("hub.verify", "sync") fu, _ := url.Parse(f.Url) fu.Fragment = "" u.Add("hub.topic", fu.String()) req, err := http.NewRequest("POST", PUBSUBHUBBUB_HUB, strings.NewReader(u.Encode())) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") cl := urlfetch.Client(c) resp, err := cl.Do(req) if err != nil { c.Errorf("req error: %v", err) } else if resp.StatusCode != 204 { c.Errorf("resp: %v - %v", f.Url, resp.Status) c.Errorf("%s", resp.Body) } else { c.Infof("subscribed: %v", f.Url) } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) retry, _ := strconv.Atoi(r.FormValue("retry")) c.Errorf("retry: %v", retry) //* iterator it := gn.Run(q) i := 0 done := false for { k, err := it.Next(nil) if err == datastore.Done { done = true break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } i++ } c.Infof("updating %d feeds", i) fmt.Fprintf(w, "updating %d feeds", i) if !done { time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{ "retry": {strconv.Itoa(retry + 1)}, }) if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil { c.Errorf("taskqueue update feeds error: %v", err.Error()) } c.Errorf("ran update again") fmt.Fprintf(w, "\nran update again") } //*/ /* get all q = q.Limit(1000) keys, _ := gn.GetAll(q, nil) for _, k := range keys { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) fmt.Fprintf(w, "updating %d feeds", len(keys)) //*/ }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) q = q.Limit(1000) it := gn.Run(q) var keys []*datastore.Key var del []*datastore.Key for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } else if len(k.StringID()) == 0 { del = append(del, k) continue } keys = append(keys, k) } tasks := make([]*taskqueue.Task, len(keys)) for i, k := range keys { tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) if len(del) > 0 { c.Errorf("attempt to delete %v feeds", len(del)) if err := gn.DeleteMulti(del); err != nil { c.Errorf("delete error: %v", err.Error()) } fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`) fmt.Fprintf(w, "attempt to delete %v feeds", len(del)) for _, k := range del { fmt.Fprintf(w, "\n<br>%v", k) } } fmt.Fprintf(w, "updating %d feeds", len(keys)) }
func ClearFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { if !isDevServer { return } cu := user.Current(c) gn := goon.FromContext(c) done := make(chan bool) go func() { u := &User{Id: cu.ID} defer func() { done <- true }() ud := &UserData{Id: "data", Parent: gn.Key(u)} if err := gn.Get(u); err != nil { c.Errorf("user del err: %v", err.Error()) return } gn.Get(ud) u.Read = time.Time{} ud.Read = nil ud.Opml = nil gn.PutMany(u, ud) c.Infof("%v cleared", u.Email) }() del := func(kind string) { defer func() { done <- true }() q := datastore.NewQuery(kind).KeysOnly() keys, err := gn.GetAll(q, nil) if err != nil { c.Errorf("err: %v", err.Error()) return } if err := gn.DeleteMulti(keys); err != nil { c.Errorf("err: %v", err.Error()) return } c.Infof("%v deleted", kind) } types := []interface{}{ &Feed{}, &Story{}, &StoryContent{}, &DateFormat{}, &Log{}, &UserOpml{}, } for _, i := range types { k := gn.Key(i).Kind() go del(k) } for i := 0; i < len(types); i++ { <-done } http.Redirect(w, r, routeUrl("main"), http.StatusFound) }
// Task used to subscribe a feed to push. func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { start := time.Now() gn := goon.FromContext(c) f := Feed{Url: r.FormValue("feed")} fk := gn.Key(&f) s := "" defer func() { gn.Put(&Log{ Parent: fk, Id: time.Now().UnixNano(), Text: "SubscribeFeed - start " + start.String() + " - f.sub " + f.Subscribed.String() + " - " + s, }) }() if err := gn.Get(&f); err != nil { c.Errorf("%v: %v", err, f.Url) serveError(w, err) s += "err" return } else if f.IsSubscribed() { s += "is subscribed" return } u := url.Values{} u.Add("hub.callback", f.PubSubURL()) u.Add("hub.mode", "subscribe") u.Add("hub.verify", "sync") fu, _ := url.Parse(f.Url) fu.Fragment = "" u.Add("hub.topic", fu.String()) req, err := http.NewRequest("POST", f.Hub, strings.NewReader(u.Encode())) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") cl := &http.Client{ Transport: &urlfetch.Transport{ Context: c, Deadline: time.Minute, }, } resp, err := cl.Do(req) if err != nil { c.Errorf("req error: %v", err) } else if resp.StatusCode != http.StatusNoContent { f.Subscribed = time.Now().Add(time.Hour * 48) gn.Put(&f) if resp.StatusCode != http.StatusConflict { c.Errorf("resp: %v - %v", f.Url, resp.Status) c.Errorf("%s", resp.Body) } s += "resp err" } else { c.Infof("subscribed: %v", f.Url) s += "success" } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) keys, _ := gn.GetAll(q, nil) for _, k := range keys { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) if _, err := taskqueue.Add(c, t, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { q := datastore.NewQuery("F").KeysOnly() q = q.Filter("n <=", time.Now()) q = q.Limit(3000) var keys []*datastore.Key it := q.Run(c) retry := 0 for { k, err := it.Next(nil) if err != nil { c.Errorf("next error: %v, retry: %v", err.Error(), retry) if retry == 5 { break } retry++ it = q.Run(c) } else { keys = append(keys, k) } } if len(keys) == 0 { c.Errorf("giving up") return } tasks := make([]*taskqueue.Task, len(keys)) for i, k := range keys { c.Infof("task: %v, %v", i, k) tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) }
func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error { gn := goon.FromContext(c) o := outline.Outline[0] c.Infof("adding feed %v to user %s", o.XmlUrl, userid) fu, ferr := url.Parse(o.XmlUrl) if ferr != nil { return ferr } fu.Fragment = "" o.XmlUrl = fu.String() f := Feed{Url: o.XmlUrl} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { if feed, stories, err := fetchFeed(c, o.XmlUrl, o.XmlUrl); err != nil { return fmt.Errorf("could not add feed %s: %v", o.XmlUrl, err) } else { f = *feed f.Updated = time.Time{} f.Checked = f.Updated f.NextUpdate = f.Updated f.LastViewed = time.Now() gn.Put(&f) for _, s := range stories { s.Created = s.Published } if err := updateFeed(c, f.Url, feed, stories, false, false, false); err != nil { return err } o.XmlUrl = feed.Url o.HtmlUrl = feed.Link if o.Title == "" { o.Title = feed.Title } } } else if err != nil { return err } else { o.HtmlUrl = f.Link if o.Title == "" { o.Title = f.Title } } o.Text = "" return nil }
func AdminUser(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Kind(&User{})).Limit(1) q = q.Filter("e =", r.FormValue("u")) it := gn.Run(q) var u User ud := UserData{Id: "data"} var h []Log k, err := it.Next(&u) if err != nil { serveError(w, err) return } ud.Parent = gn.Key(&u) gn.Get(&ud) until := r.FormValue("until") if d, err := time.Parse("2006-01-02", until); err == nil { u.Until = d gn.Put(&u) } if o := []byte(r.FormValue("opml")); len(o) > 0 { opml := Opml{} if err := json.Unmarshal(o, &opml); err != nil { serveError(w, err) return } ud.Opml = o if _, err := gn.Put(&ud); err != nil { serveError(w, err) return } c.Infof("opml updated") } q = datastore.NewQuery(gn.Kind(&Log{})).Ancestor(k) _, err = gn.GetAll(q, &h) if err := templates.ExecuteTemplate(w, "admin-user.html", struct { User User Data UserData Log []Log }{ u, ud, h, }); err != nil { serveError(w, err) } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now()) q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron var keys []*datastore.Key it := q.Run(appengine.Timeout(c, time.Second*60)) for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } keys = append(keys, k) } if len(keys) == 0 { c.Errorf("no results") return } c.Infof("updating %d feeds", len(keys)) var tasks []*taskqueue.Task for _, k := range keys { tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, })) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } }
func taskSender(c mpg.Context, queue string, tc chan *taskqueue.Task, done chan bool) { const taskLimit = 100 tasks := make([]*taskqueue.Task, 0, taskLimit) send := func() { taskqueue.AddMulti(c, tasks, queue) c.Infof("added %v tasks", len(tasks)) tasks = tasks[0:0] } for t := range tc { tasks = append(tasks, t) if len(tasks) == taskLimit { send() } } if len(tasks) > 0 { send() } done <- true }
func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error { gn := goon.FromContext(c) o := outline.Outline[0] c.Infof("adding feed %v to user %s", o.XmlUrl, userid) f := Feed{Url: o.XmlUrl} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { if feed, stories := fetchFeed(c, o.XmlUrl, o.XmlUrl); feed == nil { return errors.New(fmt.Sprintf("could not add feed %s", o.XmlUrl)) } else { f = *feed f.Updated = time.Time{} f.Checked = f.Updated f.NextUpdate = f.Updated if strings.TrimSpace(f.Url) == "" { c.Criticalf("badurl4: %v, %v", o.XmlUrl, o) return errors.New("badurl4") } gn.PutComplete(&f) if err := updateFeed(c, f.Url, feed, stories); err != nil { return err } o.XmlUrl = feed.Url o.HtmlUrl = feed.Link if o.Title == "" { o.Title = feed.Title } } } else if err != nil { return err } else { o.HtmlUrl = f.Link if o.Title == "" { o.Title = f.Title } } o.Text = "" return nil }
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed %s", url) last := len(r.FormValue("last")) > 0 f := Feed{Url: url} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { c.Errorf("no such entity") return } else if err != nil { return } else if last { // noop } else if time.Now().Before(f.NextUpdate) { c.Infof("feed %v already updated: %v", url, f.NextUpdate) return } f.Subscribe(c) feedError := func(err error) { f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err) } if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil { if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil { feedError(err) } } else { feedError(err) } }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly() q = q.Filter("n <=", time.Now()) q = q.Limit(2500) it := gn.Run(q) var keys []*datastore.Key for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } keys = append(keys, k) } tasks := make([]*taskqueue.Task, len(keys)) for i, k := range keys { tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) fmt.Fprintf(w, "updating %d feeds", len(keys)) }
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { q := datastore.NewQuery("F").KeysOnly() q = q.Filter("n <=", time.Now()) q = q.Limit(3000) var keys []*datastore.Key for i := 0; i < 5; i++ { if _keys, err := q.GetAll(c, nil); err != nil { c.Errorf("get all error: %v, retry %v", err.Error(), i) } else { c.Errorf("got %v keys", len(_keys)) keys = _keys break } } if len(keys) == 0 { c.Errorf("giving up") return } tasks := make([]*taskqueue.Task, len(keys)) for i, k := range keys { tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {k.StringID()}, }) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } c.Infof("updating %d feeds", len(keys)) }
func Account(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := User{Id: cu.ID} uc := &UserCharge{Id: 1, Parent: gn.Key(&u)} if err := gn.Get(uc); err == nil { if uc.Next.Before(time.Now()) { if resp, err := stripe(c, "GET", "customers/"+uc.Customer, ""); err == nil { if nuc, err := setCharge(c, resp); err == nil { uc = nuc c.Infof("updated user charge %v", cu.ID) } } } b, _ := json.Marshal(&uc) w.Write(b) } else if err != datastore.ErrNoSuchEntity { serveError(w, err) return } }
func CFix(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Infof("fix feed %s", url) f := Feed{Url: url} if err := gn.Get(&f); err != nil { c.Criticalf("cfix err: %v", err) serveError(w, err) return } q := datastore.NewQuery("S").Ancestor(gn.Key(&f)) var ss []*Story keys, err := q.GetAll(c, &ss) if err != nil { c.Errorf("getall err: %v", err) serveError(w, err) return } c.Infof("trying to fix %v stories", len(ss)) const putLimit = 500 for i := 0; i <= len(keys)/putLimit; i++ { lo := i * putLimit hi := (i + 1) * putLimit if hi > len(keys) { hi = len(keys) } c.Infof("%v - %v", lo, hi) if _, err := datastore.PutMulti(c, keys[lo:hi], ss[lo:hi]); err != nil { c.Errorf("err: %v, %v, %v", lo, hi, err) } } }
func BackendStart(c mpg.Context, w http.ResponseWriter, r *http.Request) { const sz = 100 ic := 0 gn := goon.FromContext(c) fk := gn.Key(&Feed{Url: "a"}) q := datastore.NewQuery("F").Filter("__key__ <", fk).Order("__key__").KeysOnly().Limit(1) keys, _ := q.GetAll(c, nil) if len(keys) == 0 { return } c.Errorf("start: %v", keys[0]) startid := keys[0].IntID() / sz var f func(appengine.Context) f = func(c appengine.Context) { c.Errorf("new request: %d", ic) t1 := time.Now() wg := sync.WaitGroup{} wg.Add(sz) var j int64 for j = 0; j < sz; j++ { go func(j int64) { k := datastore.NewKey(c, "F", "", startid*sz+j, nil) c.Infof("del: %v", k) if err := datastore.Delete(c, k); err != nil { c.Errorf("delete err: %v", err.Error()) } wg.Done() }(j) } wg.Wait() t2 := time.Now() c.Infof("%v, %v, %v", t1, t2, t2.Sub(t1)) ic++ startid++ runtime.RunInBackground(c, f) } runtime.RunInBackground(c, f) }
func Search(c mp.Context, w http.ResponseWriter, r *http.Request) error { query := r.URL.Query() q := query.Get("q") if q == "" { return core.Render(w, "songs.search.tmpl.html", &SongSearchResult{Includes: core.NewIncludes(c)}) } searchQuery := m.Wordify(q) docList, docIDs, err := data.SearchSongs(c, searchQuery) if err != nil { return err } if len(docList) == 0 { c.Infof("AND query [%s] return nothing", searchQuery) orQuery := strings.Join(strings.Fields(searchQuery), " OR ") c.Infof("Searching with OR query [%s]", orQuery) docList, docIDs, err = data.SearchSongs(c, orQuery) if err != nil { return err } } items := make([]*SongSearchResultItem, 0) for i, _ := range docList { items = append(items, createSongSearchResultItem(q, docIDs[i], docList[i])) } result := &SongSearchResult{ Includes: core.NewIncludes(c), Query: q, Items: items, } return core.Render(w, "songs.search.tmpl.html", result) }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) furl := r.FormValue("feed") oldURL := false if len(furl) == 0 { vars := mux.Vars(r) furl = vars["feed"] oldURL = true } b, _ := base64.URLEncoding.DecodeString(furl) f := Feed{Url: string(b)} c.Infof("url: %v", f.Url) if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if oldURL { c.Warningf("old url") http.Error(w, "", http.StatusNotFound) return } if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.Put(&f) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else if !f.NotViewed() { c.Infof("push: %v", f.Url) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss, err := ParseFeed(c, f.Url, b) if err != nil { c.Errorf("parse error: %v", err) return } if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil { c.Errorf("push error: %v", err) } } else { c.Infof("not viewed") } }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) furl := r.FormValue("feed") b, _ := base64.URLEncoding.DecodeString(furl) f := Feed{Url: string(b)} c.Infof("url: %v", f.Url) if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.PutMany(&f, &Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "SubscribeCallback - subscribed - " + f.Subscribed.String(), }) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else if !f.NotViewed() { c.Infof("push: %v", f.Url) gn.Put(&Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "SubscribeCallback - push update", }) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss, err := ParseFeed(c, f.Url, f.Url, b) if err != nil { c.Errorf("parse error: %v", err) return } if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil { c.Errorf("push error: %v", err) } } else { c.Infof("not viewed") } }
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error { gn := goon.FromContext(c) f := Feed{Url: url} if err := gn.Get(&f); err != nil { return errors.New(fmt.Sprintf("feed not found: %s", url)) } // Compare the feed's listed update to the story's update. // Note: these may not be accurate, hence, only compare them to each other, // since they should have the same relative error. storyDate := f.Updated hasUpdated := !feed.Updated.IsZero() isFeedUpdated := f.Updated == feed.Updated if !hasUpdated { feed.Updated = f.Updated } feed.Date = f.Date f = *feed if hasUpdated && isFeedUpdated { c.Infof("feed %s already updated to %v, putting", url, feed.Updated) f.Updated = time.Now() if strings.TrimSpace(f.Url) == "" { c.Criticalf("badurl5: %v, %v", url, f) return errors.New("badurl5") } gn.PutComplete(&f) return nil } c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate) var newStories []*Story for _, s := range stories { if s.Updated.IsZero() || !s.Updated.Before(storyDate) { newStories = append(newStories, s) } } c.Debugf("%v possible stories to update", len(newStories)) puts := []interface{}{&f} // find non existant stories fk := gn.Key(&f) getStories := make([]*Story, len(newStories)) for i, s := range newStories { getStories[i] = &Story{Id: s.Id, Parent: fk} } err := gn.GetMulti(getStories) if _, ok := err.(appengine.MultiError); err != nil && !ok { c.Errorf("get multi error: %v", err.Error()) return err } var updateStories []*Story for i, s := range getStories { if goon.NotFound(err, i) { updateStories = append(updateStories, newStories[i]) } else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) { newStories[i].Created = s.Created newStories[i].Published = s.Published updateStories = append(updateStories, newStories[i]) } } c.Debugf("%v update stories", len(updateStories)) for _, s := range updateStories { puts = append(puts, s) gn.Put(&StoryContent{ Id: 1, Parent: gn.Key(s), Content: s.content, }) } c.Debugf("putting %v entities", len(puts)) if len(puts) > 1 { f.Date = time.Now() if !hasUpdated { f.Updated = f.Date } } if f.Url == "" { c.Criticalf("badurl6: %v", f) return errors.New("badurl6") } gn.PutMultiComplete(puts) return nil }
func CFixer(c mpg.Context, w http.ResponseWriter, r *http.Request) { q := datastore.NewQuery("F").KeysOnly() q = q.Limit(1000) cs := r.FormValue("c") if len(cs) > 0 { if cur, err := datastore.DecodeCursor(cs); err == nil { q = q.Start(cur) c.Infof("starting at %v", cur) } else { c.Errorf("cursor error %v", err.Error()) } } var keys []*datastore.Key it := q.Run(Timeout(c, time.Second*15)) for { k, err := it.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("next error: %v", err.Error()) break } keys = append(keys, k) } if len(keys) == 0 { c.Errorf("no results") return } else { cur, err := it.Cursor() if err != nil { c.Errorf("to cur error %v", err.Error()) } else { c.Infof("add with cur %v", cur) t := taskqueue.NewPOSTTask("/tasks/cfixer", url.Values{ "c": {cur.String()}, }) taskqueue.Add(c, t, "cfixer") } } c.Infof("fixing %d feeds", len(keys)) var tasks []*taskqueue.Task for _, k := range keys { c.Infof("f: %v", k.StringID()) tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/cfix", url.Values{ "feed": {k.StringID()}, })) } var ts []*taskqueue.Task const taskLimit = 100 for len(tasks) > 0 { if len(tasks) > taskLimit { ts = tasks[:taskLimit] tasks = tasks[taskLimit:] } else { ts = tasks tasks = tasks[0:0] } if _, err := taskqueue.AddMulti(c, ts, "cfixer"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } } }
func ImportOpmlTask(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) userid := r.FormValue("user") bk := r.FormValue("key") fr := blobstore.NewReader(c, appengine.BlobKey(bk)) var skip int if s, err := strconv.Atoi(r.FormValue("skip")); err == nil { skip = s } c.Debugf("reader import for %v, skip %v", userid, skip) var userOpml []*OpmlOutline remaining := skip var proc func(label string, outlines []*OpmlOutline) proc = func(label string, outlines []*OpmlOutline) { for _, o := range outlines { if o.Title == "" { o.Title = o.Text } if o.XmlUrl != "" { if remaining > 0 { remaining-- } else if len(userOpml) < IMPORT_LIMIT { userOpml = append(userOpml, &OpmlOutline{ Title: label, Outline: []*OpmlOutline{o}, }) } } if o.Title != "" && len(o.Outline) > 0 { proc(o.Title, o.Outline) } } } opml := Opml{} d := xml.NewDecoder(fr) d.CharsetReader = charset.NewReader d.Strict = false if err := d.Decode(&opml); err != nil { c.Errorf("opml error: %v", err.Error()) return } proc("", opml.Outline) // todo: refactor below with similar from ImportReaderTask wg := sync.WaitGroup{} wg.Add(len(userOpml)) for i := range userOpml { go func(i int) { o := userOpml[i].Outline[0] if err := addFeed(c, userid, userOpml[i]); err != nil { c.Warningf("opml import error: %v", err.Error()) // todo: do something here? } c.Debugf("opml import: %s, %s", o.Title, o.XmlUrl) wg.Done() }(i) } wg.Wait() ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})} if err := gn.RunInTransaction(func(gn *goon.Goon) error { gn.Get(&ud) mergeUserOpml(&ud, userOpml...) _, err := gn.Put(&ud) return err }, nil); err != nil { w.WriteHeader(http.StatusInternalServerError) c.Errorf("ude update error: %v", err.Error()) return } if len(userOpml) == IMPORT_LIMIT { task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{ "key": {bk}, "user": {userid}, "skip": {strconv.Itoa(skip + IMPORT_LIMIT)}, }) taskqueue.Add(c, task, "import-reader") } else { c.Infof("opml import done: %v", userid) } }
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error { gn := goon.FromContext(c) f := Feed{Url: url} if err := gn.Get(&f); err != nil { return fmt.Errorf("feed not found: %s", url) } // Compare the feed's listed update to the story's update. // Note: these may not be accurate, hence, only compare them to each other, // since they should have the same relative error. storyDate := f.Updated hasUpdated := !feed.Updated.IsZero() isFeedUpdated := f.Updated == feed.Updated if !hasUpdated { feed.Updated = f.Updated } feed.Date = f.Date f = *feed if hasUpdated && isFeedUpdated { c.Infof("feed %s already updated to %v, putting", url, feed.Updated) f.Updated = time.Now() gn.Put(&f) return nil } c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate) var newStories []*Story for _, s := range stories { if s.Updated.IsZero() || !s.Updated.Before(storyDate) { newStories = append(newStories, s) } } c.Debugf("%v possible stories to update", len(newStories)) puts := []interface{}{&f} // find non existant stories fk := gn.Key(&f) getStories := make([]*Story, len(newStories)) for i, s := range newStories { getStories[i] = &Story{Id: s.Id, Parent: fk} } err := gn.GetMulti(getStories) if _, ok := err.(appengine.MultiError); err != nil && !ok { c.Errorf("get multi error: %v", err.Error()) return err } var updateStories []*Story for i, s := range getStories { if goon.NotFound(err, i) { updateStories = append(updateStories, newStories[i]) } else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) { newStories[i].Created = s.Created newStories[i].Published = s.Published updateStories = append(updateStories, newStories[i]) } } c.Debugf("%v update stories", len(updateStories)) for _, s := range updateStories { puts = append(puts, s) sc := StoryContent{ Id: 1, Parent: gn.Key(s), } buf := &bytes.Buffer{} if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil { gz.Write([]byte(s.content)) gz.Close() sc.Compressed = buf.Bytes() } if len(sc.Compressed) == 0 { sc.Content = s.content } gn.Put(&sc) } c.Debugf("putting %v entities", len(puts)) if len(puts) > 1 { f.Date = time.Now() if !hasUpdated { f.Updated = f.Date } } gn.PutMulti(puts) return nil }