func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed %s", url) f := Feed{Url: url} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { return } else if time.Now().Before(f.NextUpdate) { c.Infof("feed %v already updated", url) return } if feed, stories := fetchFeed(c, url, url); feed != nil { updateFeed(c, url, feed, stories) } else { f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate) } }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) vars := mux.Vars(r) b, _ := base64.URLEncoding.DecodeString(vars["feed"]) f := Feed{Url: string(b)} if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.Put(&f) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else { c.Infof("push: %v", f.Url) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss := ParseFeed(c, f.Url, b) err := updateFeed(c, f.Url, nf, ss) if err != nil { c.Errorf("push error: %v", err) } } }
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(appengine.Timeout(c, time.Minute)) url := r.FormValue("feed") if url == "" { c.Errorf("empty update feed") return } c.Debugf("update feed %s", url) last := len(r.FormValue("last")) > 0 f := Feed{Url: url} s := "" defer func() { gn.Put(&Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "UpdateFeed - " + s, }) }() if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { c.Errorf("no such entity - " + url) s += "NSE" return } else if err != nil { s += "err - " + err.Error() return } else if last { // noop } if time.Now().Before(f.NextUpdate) { c.Errorf("feed %v already updated: %v", url, f.NextUpdate) s += "already updated" return } feedError := func(err error) { s += "feed err - " + err.Error() f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err) } if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil { if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil { feedError(err) } else { s += "success" } } else { feedError(err) } f.Subscribe(c) }
func UpdateFeedLast(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed last %s", url) f := Feed{Url: url} if err := gn.Get(&f); err != nil { return } f.LastViewed = time.Now() gn.Put(&f) }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) furl := r.FormValue("feed") oldURL := false if len(furl) == 0 { vars := mux.Vars(r) furl = vars["feed"] oldURL = true } b, _ := base64.URLEncoding.DecodeString(furl) f := Feed{Url: string(b)} c.Infof("url: %v", f.Url) if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if oldURL { c.Warningf("old url") http.Error(w, "", http.StatusNotFound) return } if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.Put(&f) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else if !f.NotViewed() { c.Infof("push: %v", f.Url) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss, err := ParseFeed(c, f.Url, b) if err != nil { c.Errorf("parse error: %v", err) return } if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil { c.Errorf("push error: %v", err) } } else { c.Infof("not viewed") } }
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) furl := r.FormValue("feed") b, _ := base64.URLEncoding.DecodeString(furl) f := Feed{Url: string(b)} c.Infof("url: %v", f.Url) if err := gn.Get(&f); err != nil { http.Error(w, "", http.StatusNotFound) return } if r.Method == "GET" { if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url { http.Error(w, "", http.StatusNotFound) return } w.Write([]byte(r.FormValue("hub.challenge"))) i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds")) f.Subscribed = time.Now().Add(time.Second * time.Duration(i)) gn.PutMany(&f, &Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "SubscribeCallback - subscribed - " + f.Subscribed.String(), }) c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed) return } else if !f.NotViewed() { c.Infof("push: %v", f.Url) gn.Put(&Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "SubscribeCallback - push update", }) defer r.Body.Close() b, _ := ioutil.ReadAll(r.Body) nf, ss, err := ParseFeed(c, f.Url, f.Url, b) if err != nil { c.Errorf("parse error: %v", err) return } if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil { c.Errorf("push error: %v", err) } } else { c.Infof("not viewed") } }
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) url := r.FormValue("feed") c.Debugf("update feed %s", url) last := len(r.FormValue("last")) > 0 f := Feed{Url: url} if err := gn.Get(&f); err == datastore.ErrNoSuchEntity { c.Errorf("no such entity") return } else if err != nil { return } else if last { // noop } else if time.Now().Before(f.NextUpdate) { c.Infof("feed %v already updated: %v", url, f.NextUpdate) return } f.Subscribe(c) feedError := func(err error) { f.Errors++ v := f.Errors + 1 const max = 24 * 7 if v > max { v = max } else if f.Errors == 1 { v = 0 } f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v)) gn.Put(&f) c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err) } if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil { if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil { feedError(err) } } else { feedError(err) } }
func ImportReaderTask(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) userid := r.FormValue("user") bk := r.FormValue("key") fr := blobstore.NewReader(c, appengine.BlobKey(bk)) data, err := ioutil.ReadAll(fr) if err != nil { return } var skip int if s, err := strconv.Atoi(r.FormValue("skip")); err == nil { skip = s } v := struct { Subscriptions []struct { Id string `json:"id"` Title string `json:"title"` HtmlUrl string `json:"htmlUrl"` Categories []struct { Id string `json:"id"` Label string `json:"label"` } `json:"categories"` } `json:"subscriptions"` }{} json.Unmarshal(data, &v) c.Debugf("reader import for %v, skip %v, len %v", userid, skip, len(v.Subscriptions)) end := skip + IMPORT_LIMIT if end > len(v.Subscriptions) { end = len(v.Subscriptions) } wg := sync.WaitGroup{} wg.Add(end - skip) userOpml := make([]*OpmlOutline, end-skip) for i := range v.Subscriptions[skip:end] { go func(i int) { sub := v.Subscriptions[skip+i] var label string if len(sub.Categories) > 0 { label = sub.Categories[0].Label } outline := &OpmlOutline{ Title: label, Outline: []*OpmlOutline{ &OpmlOutline{ XmlUrl: sub.Id[5:], Title: sub.Title, }, }, } userOpml[i] = outline if err := addFeed(c, userid, outline); err != nil { c.Warningf("reader import error: %v", err.Error()) // todo: do something here? } c.Debugf("reader import: %s, %s", sub.Title, sub.Id) wg.Done() }(i) } wg.Wait() ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})} if err := gn.RunInTransaction(func(gn *goon.Goon) error { gn.Get(&ud) mergeUserOpml(&ud, userOpml...) _, err := gn.Put(&ud) return err }, nil); err != nil { w.WriteHeader(http.StatusInternalServerError) c.Errorf("ude update error: %v", err.Error()) return } if end < len(v.Subscriptions) { task := taskqueue.NewPOSTTask(routeUrl("import-reader-task"), url.Values{ "key": {bk}, "user": {userid}, "skip": {strconv.Itoa(skip + IMPORT_LIMIT)}, }) taskqueue.Add(c, task, "import-reader") } else { blobstore.Delete(c, appengine.BlobKey(bk)) } }
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error { gn := goon.FromContext(c) f := Feed{Url: url} if err := gn.Get(&f); err != nil { return fmt.Errorf("feed not found: %s", url) } // Compare the feed's listed update to the story's update. // Note: these may not be accurate, hence, only compare them to each other, // since they should have the same relative error. storyDate := f.Updated hasUpdated := !feed.Updated.IsZero() isFeedUpdated := f.Updated == feed.Updated if !hasUpdated { feed.Updated = f.Updated } feed.Date = f.Date f = *feed if hasUpdated && isFeedUpdated { c.Infof("feed %s already updated to %v, putting", url, feed.Updated) f.Updated = time.Now() gn.Put(&f) return nil } c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate) var newStories []*Story for _, s := range stories { if s.Updated.IsZero() || !s.Updated.Before(storyDate) { newStories = append(newStories, s) } } c.Debugf("%v possible stories to update", len(newStories)) puts := []interface{}{&f} // find non existant stories fk := gn.Key(&f) getStories := make([]*Story, len(newStories)) for i, s := range newStories { getStories[i] = &Story{Id: s.Id, Parent: fk} } err := gn.GetMulti(getStories) if _, ok := err.(appengine.MultiError); err != nil && !ok { c.Errorf("get multi error: %v", err.Error()) return err } var updateStories []*Story for i, s := range getStories { if goon.NotFound(err, i) { updateStories = append(updateStories, newStories[i]) } else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) { newStories[i].Created = s.Created newStories[i].Published = s.Published updateStories = append(updateStories, newStories[i]) } } c.Debugf("%v update stories", len(updateStories)) for _, s := range updateStories { puts = append(puts, s) sc := StoryContent{ Id: 1, Parent: gn.Key(s), } buf := &bytes.Buffer{} if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil { gz.Write([]byte(s.content)) gz.Close() sc.Compressed = buf.Bytes() } if len(sc.Compressed) == 0 { sc.Content = s.content } gn.Put(&sc) } c.Debugf("putting %v entities", len(puts)) if len(puts) > 1 { f.Date = time.Now() if !hasUpdated { f.Updated = f.Date } } gn.PutMulti(puts) return nil }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) read := make(Read) var uf Opml c.Step("unmarshal user data", func() { json.Unmarshal(ud.Read, &read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) c.Step("feed fetch + wait", func() { queue := make(chan *Feed) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var newStories []*Story if u.Read.Before(f.Date) { c.Debugf("query for %v", f.Url) fk := gn.Key(f) sq := q.Ancestor(fk).Filter("p >", u.Read).KeysOnly().Order("-p") keys, _ := gn.GetAll(sq, nil) stories := make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) for _, st := range stories { found := false for _, s := range read[f.Url] { if s == st.Id { found = true break } } if !found { newStories = append(newStories, st) } } } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true c.Debugf("fixing link %v, %v -> %v", f.Url, opmlMap[f.Url].HtmlUrl, f.Link) opmlMap[f.Url].HtmlUrl = f.Link } lock.Lock() fl[f.Url] = newStories if len(newStories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() } } for i := 0; i < 20; i++ { go feedProc() } wg.Add(len(feeds)) for _, f := range feeds { queue <- f } close(queue) wg.Wait() }) if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) u.Read = last ud.Read = nil gn.PutMany(u, ud) } } if updatedLinks { ud.Opml, _ = json.Marshal(&uf) gn.Put(ud) } c.Step("json marshal", func() { b, _ := json.Marshal(struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, }) w.Write(b) }) }
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story, updateAll, fromSub, updateLast bool) error { gn := goon.FromContext(c) f := Feed{Url: url} if err := gn.Get(&f); err != nil { return fmt.Errorf("feed not found: %s", url) } gn.Put(&Log{ Parent: gn.Key(&f), Id: time.Now().UnixNano(), Text: "feed update", }) // Compare the feed's listed update to the story's update. // Note: these may not be accurate, hence, only compare them to each other, // since they should have the same relative error. storyDate := f.Updated hasUpdated := !feed.Updated.IsZero() isFeedUpdated := f.Updated.Equal(feed.Updated) if !hasUpdated { feed.Updated = f.Updated } feed.Date = f.Date feed.Average = f.Average feed.LastViewed = f.LastViewed f = *feed if updateLast { f.LastViewed = time.Now() } if hasUpdated && isFeedUpdated && !updateAll && !fromSub { c.Infof("feed %s already updated to %v, putting", url, feed.Updated) f.Updated = time.Now() scheduleNextUpdate(&f) gn.Put(&f) return nil } c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v, stories: %v", hasUpdated, isFeedUpdated, storyDate, len(stories)) puts := []interface{}{&f} // find non existant stories fk := gn.Key(&f) getStories := make([]*Story, len(stories)) for i, s := range stories { getStories[i] = &Story{Id: s.Id, Parent: fk} } err := gn.GetMulti(getStories) if _, ok := err.(appengine.MultiError); err != nil && !ok { c.Errorf("get multi error: %v", err.Error()) return err } var updateStories []*Story for i, s := range getStories { if goon.NotFound(err, i) { updateStories = append(updateStories, stories[i]) } else if (!stories[i].Updated.IsZero() && !stories[i].Updated.Equal(s.Updated)) || updateAll { stories[i].Created = s.Created stories[i].Published = s.Published updateStories = append(updateStories, stories[i]) } } c.Debugf("%v update stories", len(updateStories)) for _, s := range updateStories { puts = append(puts, s) sc := StoryContent{ Id: 1, Parent: gn.Key(s), } buf := &bytes.Buffer{} if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil { gz.Write([]byte(s.content)) gz.Close() sc.Compressed = buf.Bytes() } if len(sc.Compressed) == 0 { sc.Content = s.content } gn.Put(&sc) } c.Debugf("putting %v entities", len(puts)) if len(puts) > 1 { updateAverage(&f, f.Date, len(puts)-1) f.Date = time.Now() if !hasUpdated { f.Updated = f.Date } } scheduleNextUpdate(&f) if fromSub { wait := time.Now().Add(time.Hour * 6) if f.NextUpdate.Before(wait) { f.NextUpdate = time.Now().Add(time.Hour * 6) } } delay := f.NextUpdate.Sub(time.Now()) c.Infof("next update scheduled for %v from now", delay-delay%time.Second) gn.PutMulti(puts) return nil }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) put := false fixRead := false if time.Since(u.Read) > oldDuration { c.Warningf("u.Read too old, fixing: %v", u.Read) u.Read = time.Now().Add(-oldDuration) put = true fixRead = true c.Warningf("new: %v", u.Read) } read := make(Read) var uf Opml c.Step("unmarshal user data", func() { gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) now := time.Now() numStories := 0 c.Step("feed fetch + wait", func() { queue := make(chan *Feed) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var stories []*Story if u.Read.Before(f.Date) { fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories = make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } if f.Errors == 0 && f.NextUpdate.Before(now) { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {f.Url}, }) if _, err := taskqueue.Add(c, t, "update-manual"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } else { c.Warningf("manual feed update: %v", f.Url) } } f.Subscribe(c) lock.Lock() fl[f.Url] = stories numStories += len(stories) if len(stories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() } } for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) wg.Wait() }) if numStories > numStoriesLimit { c.Step("numStories", func() { c.Errorf("too many stories: %v", numStories) stories := make([]*Story, 0, numStories) for _, v := range fl { stories = append(stories, v...) } sort.Sort(sort.Reverse(Stories(stories))) last := stories[numStoriesLimit].Created stories = stories[:numStoriesLimit] u.Read = last put = true fixRead = true fl = make(map[string][]*Story) for _, s := range stories { fk := s.Parent.StringID() p := fl[fk] fl[fk] = append(p, s) } c.Errorf("filtered: %v, %v", len(stories), last) }) } if fixRead { c.Step("fix read", func() { nread := make(Read) for k, v := range fl { for _, s := range v { rs := readStory{Feed: k, Story: s.Id} if read[rs] { nread[rs] = true } } } c.Errorf("fix read: %v -> %v", len(read), len(nread)) read = nread var b bytes.Buffer gob.NewEncoder(&b).Encode(&read) ud.Read = b.Bytes() put = true }) } for k, v := range fl { newStories := make([]*Story, 0, len(v)) for _, s := range v { if !read[readStory{Feed: k, Story: s.Id}] { newStories = append(newStories, s) } } fl[k] = newStories } if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) put = true u.Read = last ud.Read = nil } } if updatedLinks { ud.Opml, _ = json.Marshal(&uf) put = true } if put { gn.PutMany(u, ud) } c.Step("json marshal", func() { o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := cleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) read := make(Read) var uf Opml c.Step("unmarshal user data", func() { json.Unmarshal(ud.Read, &read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) now := time.Now() c.Step("feed fetch + wait", func() { queue := make(chan *Feed) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var newStories []*Story if u.Read.Before(f.Date) { c.Debugf("query for %v", f.Url) fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories := make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) for _, st := range stories { found := false for _, s := range read[f.Url] { if s == st.Id { found = true break } } if !found { newStories = append(newStories, st) } } } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } if f.Errors == 0 && f.NextUpdate.Before(now) { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {f.Url}, }) if _, err := taskqueue.Add(c, t, "update-manual"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } else { c.Warningf("manual feed update: %v", f.Url) } } f.Subscribe(c) lock.Lock() fl[f.Url] = newStories if len(newStories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() } } for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) wg.Wait() }) if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) u.Read = last ud.Read = nil gn.PutMany(u, ud) } } if updatedLinks { ud.Opml, _ = json.Marshal(&uf) gn.Put(ud) } c.Step("json marshal", func() { o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := cleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) _ = utf8.RuneError }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) putU := false putUD := false fixRead := false if time.Since(u.Read) > oldDuration { u.Read = time.Now().Add(-oldDuration) putU = true fixRead = true } read := make(Read) var uf Opml c.Step("unmarshal user data", func() { gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) noads := make(map[string]bool) now := time.Now() numStories := 0 c.Step("feed fetch + wait", func() { queue := make(chan *Feed) tc := make(chan *taskqueue.Task) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var stories []*Story if u.Read.Before(f.Date) { fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories = make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } manualDone := false if time.Since(f.LastViewed) > time.Hour*24*2 { if f.NextUpdate.Equal(timeMax) { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, "last": {"1"}, }) manualDone = true } else { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-last"), url.Values{ "feed": {f.Url}, }) } } if !manualDone && now.Sub(f.NextUpdate) >= 0 { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, }) } lock.Lock() fl[f.Url] = stories numStories += len(stories) if len(stories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } if f.NoAds { noads[f.Url] = true } lock.Unlock() } } go func() { var tasks []*taskqueue.Task for t := range tc { tasks = append(tasks, t) if len(tasks) == 100 { taskqueue.AddMulti(c, tasks, "update-manual") c.Infof("added %v tasks", len(tasks)) tasks = tasks[0:0] } } if len(tasks) > 0 { taskqueue.AddMulti(c, tasks, "update-manual") c.Infof("added %v tasks", len(tasks)) } wg.Done() }() for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) // wait for feeds to complete so there are no more tasks to queue wg.Wait() wg.Add(1) // then finish enqueuing tasks close(tc) wg.Wait() }) if numStories > numStoriesLimit { c.Step("numStories", func() { c.Infof("too many stories: %v", numStories) stories := make([]*Story, 0, numStories) for _, v := range fl { stories = append(stories, v...) } sort.Sort(sort.Reverse(Stories(stories))) last := stories[numStoriesLimit].Created stories = stories[:numStoriesLimit] u.Read = last putU = true fixRead = true fl = make(map[string][]*Story) for _, s := range stories { fk := s.Parent.StringID() p := fl[fk] fl[fk] = append(p, s) } c.Infof("filtered: %v, %v", len(stories), last) }) } if fixRead { c.Step("fix read", func() { nread := make(Read) for k, v := range fl { for _, s := range v { rs := readStory{Feed: k, Story: s.Id} if read[rs] { nread[rs] = true } } } read = nread var b bytes.Buffer gob.NewEncoder(&b).Encode(&read) ud.Read = b.Bytes() putUD = true }) } for k, v := range fl { newStories := make([]*Story, 0, len(v)) for _, s := range v { if !read[readStory{Feed: k, Story: s.Id}] { newStories = append(newStories, s) } } fl[k] = newStories } if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) putU = true putUD = true u.Read = last ud.Read = nil } } if updatedLinks { backupOPML(c) if o, err := json.Marshal(&uf); err == nil { ud.Opml = o putUD = true } else { saveError(c, fmt.Sprintf("%v", uf), err) c.Errorf("json UL err: %v, %v", err, uf) } } if putU { gn.Put(u) } if putUD { gn.Put(ud) } c.Step("json marshal", func() { o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string NoAds map[string]bool Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, NoAds: noads, Options: u.Options, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := sanitizer.CleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) }
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error { gn := goon.FromContext(c) f := Feed{Url: url} if err := gn.Get(&f); err != nil { return errors.New(fmt.Sprintf("feed not found: %s", url)) } // Compare the feed's listed update to the story's update. // Note: these may not be accurate, hence, only compare them to each other, // since they should have the same relative error. storyDate := f.Updated hasUpdated := !feed.Updated.IsZero() isFeedUpdated := f.Updated == feed.Updated if !hasUpdated { feed.Updated = f.Updated } feed.Date = f.Date f = *feed if hasUpdated && isFeedUpdated { c.Infof("feed %s already updated to %v, putting", url, feed.Updated) f.Updated = time.Now() if strings.TrimSpace(f.Url) == "" { c.Criticalf("badurl5: %v, %v", url, f) return errors.New("badurl5") } gn.PutComplete(&f) return nil } c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate) var newStories []*Story for _, s := range stories { if s.Updated.IsZero() || !s.Updated.Before(storyDate) { newStories = append(newStories, s) } } c.Debugf("%v possible stories to update", len(newStories)) puts := []interface{}{&f} // find non existant stories fk := gn.Key(&f) getStories := make([]*Story, len(newStories)) for i, s := range newStories { getStories[i] = &Story{Id: s.Id, Parent: fk} } err := gn.GetMulti(getStories) if _, ok := err.(appengine.MultiError); err != nil && !ok { c.Errorf("get multi error: %v", err.Error()) return err } var updateStories []*Story for i, s := range getStories { if goon.NotFound(err, i) { updateStories = append(updateStories, newStories[i]) } else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) { newStories[i].Created = s.Created newStories[i].Published = s.Published updateStories = append(updateStories, newStories[i]) } } c.Debugf("%v update stories", len(updateStories)) for _, s := range updateStories { puts = append(puts, s) gn.Put(&StoryContent{ Id: 1, Parent: gn.Key(s), Content: s.content, }) } c.Debugf("putting %v entities", len(puts)) if len(puts) > 1 { f.Date = time.Now() if !hasUpdated { f.Updated = f.Date } } if f.Url == "" { c.Criticalf("badurl6: %v", f) return errors.New("badurl6") } gn.PutMultiComplete(puts) return nil }
func ImportOpmlTask(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) userid := r.FormValue("user") bk := r.FormValue("key") fr := blobstore.NewReader(c, appengine.BlobKey(bk)) data, err := ioutil.ReadAll(fr) if err != nil { return } var skip int if s, err := strconv.Atoi(r.FormValue("skip")); err == nil { skip = s } c.Debugf("reader import for %v, skip %v", userid, skip) var userOpml []*OpmlOutline remaining := skip var proc func(label string, outlines []*OpmlOutline) proc = func(label string, outlines []*OpmlOutline) { for _, o := range outlines { if o.XmlUrl != "" { if remaining > 0 { remaining-- } else if len(userOpml) < IMPORT_LIMIT { userOpml = append(userOpml, &OpmlOutline{ Title: label, Outline: []*OpmlOutline{o}, }) } } if o.Title != "" && len(o.Outline) > 0 { proc(o.Title, o.Outline) } } } opml := Opml{} if err := xml.Unmarshal(data, &opml); err != nil { c.Errorf("opml error: %v", err.Error()) return } proc("", opml.Outline) // todo: refactor below with similar from ImportReaderTask wg := sync.WaitGroup{} wg.Add(len(userOpml)) for i := range userOpml { go func(i int) { o := userOpml[i].Outline[0] if err := addFeed(c, userid, userOpml[i]); err != nil { c.Warningf("opml import error: %v", err.Error()) // todo: do something here? } c.Debugf("opml import: %s, %s", o.Title, o.XmlUrl) wg.Done() }(i) } wg.Wait() ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})} if err := gn.RunInTransaction(func(gn *goon.Goon) error { gn.Get(&ud) mergeUserOpml(&ud, opml.Outline...) _, err := gn.Put(&ud) return err }, nil); err != nil { w.WriteHeader(http.StatusInternalServerError) c.Errorf("ude update error: %v", err.Error()) return } if len(userOpml) == IMPORT_LIMIT { task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{ "key": {bk}, "user": {userid}, "skip": {strconv.Itoa(skip + IMPORT_LIMIT)}, }) taskqueue.Add(c, task, "import-reader") } }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} if err := gn.GetMulti([]interface{}{u, ud}); err != nil && !goon.NotFound(err, 1) { serveError(w, err) return } l := &Log{ Parent: ud.Parent, Id: time.Now().UnixNano(), Text: "list feeds", } l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml)) putU := false putUD := false fixRead := false if time.Since(u.Read) > oldDuration { u.Read = time.Now().Add(-oldDuration) putU = true fixRead = true l.Text += ", u.Read" } trialRemaining := 0 if STRIPE_KEY != "" && ud.Opml != nil { if u.Created.IsZero() { u.Created = time.Now() putU = true } else if time.Since(u.Created) > accountFreeDuration { b, _ := json.Marshal(struct { ErrorSubscription bool }{ true, }) w.Write(b) return } trialRemaining = int((accountFreeDuration-time.Since(u.Created))/time.Hour/24) + 1 } read := make(Read) var uf Opml c.Step("unmarshal user data", func(c mpg.Context) { gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func(c mpg.Context) { gn := goon.FromContext(c) for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) now := time.Now() numStories := 0 c.Step("feed fetch + wait", func(c mpg.Context) { queue := make(chan *Feed) tc := make(chan *taskqueue.Task) done := make(chan bool) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { c.Step(f.Title, func(c mpg.Context) { defer wg.Done() var stories []*Story gn := goon.FromContext(c) if !f.Date.Before(u.Read) { fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >=", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories = make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) } if f.Link != opmlMap[f.Url].HtmlUrl { l.Text += fmt.Sprintf(", link: %v -> %v", opmlMap[f.Url].HtmlUrl, f.Link) updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } manualDone := false if time.Since(f.LastViewed) > time.Hour*24*2 { if f.NextUpdate.Equal(timeMax) { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, "last": {"1"}, }) manualDone = true } else { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-last"), url.Values{ "feed": {f.Url}, }) } } if !manualDone && now.Sub(f.NextUpdate) >= 0 { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, }) } lock.Lock() fl[f.Url] = stories numStories += len(stories) if len(stories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() }) } } go taskSender(c, "update-manual", tc, done) for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) // wait for feeds to complete so there are no more tasks to queue wg.Wait() // then finish enqueuing tasks close(tc) <-done }) if numStories > 0 { c.Step("numStories", func(c mpg.Context) { stories := make([]*Story, 0, numStories) for _, v := range fl { stories = append(stories, v...) } sort.Sort(sort.Reverse(Stories(stories))) if len(stories) > numStoriesLimit { stories = stories[:numStoriesLimit] fl = make(map[string][]*Story) for _, s := range stories { fk := s.Parent.StringID() p := fl[fk] fl[fk] = append(p, s) } } last := stories[len(stories)-1].Created if !u.Read.Equal(last) { u.Read = last putU = true fixRead = true } }) } if fixRead { c.Step("fix read", func(c mpg.Context) { nread := make(Read) for k, v := range fl { for _, s := range v { rs := readStory{Feed: k, Story: s.Id} if read[rs] { nread[rs] = true } } } if len(nread) != len(read) { read = nread var b bytes.Buffer gob.NewEncoder(&b).Encode(&read) ud.Read = b.Bytes() putUD = true l.Text += ", fix read" } }) } numStories = 0 for k, v := range fl { newStories := make([]*Story, 0, len(v)) for _, s := range v { if !read[readStory{Feed: k, Story: s.Id}] { newStories = append(newStories, s) } } numStories += len(newStories) fl[k] = newStories } if numStories == 0 { l.Text += ", clear read" fixRead = false if ud.Read != nil { putUD = true ud.Read = nil } last := u.Read for _, v := range feeds { if last.Before(v.Date) { last = v.Date } } c.Infof("nothing here, move up: %v -> %v", u.Read, last) if !u.Read.Equal(last) { putU = true u.Read = last } } if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) putU = true putUD = true u.Read = last ud.Read = nil l.Text += ", read last" } } if updatedLinks { backupOPML(c) if o, err := json.Marshal(&uf); err == nil { ud.Opml = o putUD = true l.Text += ", update links" } else { c.Errorf("json UL err: %v, %v", err, uf) } } if putU { gn.Put(u) l.Text += ", putU" } if putUD { gn.Put(ud) l.Text += ", putUD" } l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml)) gn.Put(l) c.Step("json marshal", func(c mpg.Context) { gn := goon.FromContext(c) o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string TrialRemaining int }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, TrialRemaining: trialRemaining, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := sanitizer.CleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) }