func GetFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) { gn := goon.FromContext(c) f := Feed{Url: r.FormValue("f")} var stars []string wg := sync.WaitGroup{} fk := gn.Key(&f) q := datastore.NewQuery(gn.Kind(&Story{})).Ancestor(fk).KeysOnly() q = q.Order("-" + IDX_COL) if cur := r.FormValue("c"); cur != "" { if dc, err := datastore.DecodeCursor(cur); err == nil { q = q.Start(dc) } } else { // grab the stars list on the first run wg.Add(1) go c.Step("stars", func(c mpg.Context) { gn := goon.FromContext(c) usk := starKey(c, f.Url, "") q := datastore.NewQuery(gn.Kind(&UserStar{})).Ancestor(gn.Key(usk).Parent()).KeysOnly() keys, _ := gn.GetAll(q, nil) stars = make([]string, len(keys)) for i, key := range keys { stars[i] = starID(key) } wg.Done() }) } iter := gn.Run(q) var stories []*Story for i := 0; i < 20; i++ { if k, err := iter.Next(nil); err == nil { stories = append(stories, &Story{ Id: k.StringID(), Parent: k.Parent(), }) } else if err == datastore.Done { break } else { serveError(w, err) return } } cursor := "" if ic, err := iter.Cursor(); err == nil { cursor = ic.String() } gn.GetMulti(&stories) wg.Wait() b, _ := json.Marshal(struct { Cursor string Stories []*Story Stars []string `json:",omitempty"` }{ Cursor: cursor, Stories: stories, Stars: stars, }) w.Write(b) }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) read := make(Read) var uf Opml c.Step("unmarshal user data", func() { json.Unmarshal(ud.Read, &read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) c.Step("feed fetch + wait", func() { queue := make(chan *Feed) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var newStories []*Story if u.Read.Before(f.Date) { c.Debugf("query for %v", f.Url) fk := gn.Key(f) sq := q.Ancestor(fk).Filter("p >", u.Read).KeysOnly().Order("-p") keys, _ := gn.GetAll(sq, nil) stories := make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) for _, st := range stories { found := false for _, s := range read[f.Url] { if s == st.Id { found = true break } } if !found { newStories = append(newStories, st) } } } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true c.Debugf("fixing link %v, %v -> %v", f.Url, opmlMap[f.Url].HtmlUrl, f.Link) opmlMap[f.Url].HtmlUrl = f.Link } lock.Lock() fl[f.Url] = newStories if len(newStories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() } } for i := 0; i < 20; i++ { go feedProc() } wg.Add(len(feeds)) for _, f := range feeds { queue <- f } close(queue) wg.Wait() }) if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) u.Read = last ud.Read = nil gn.PutMany(u, ud) } } if updatedLinks { ud.Opml, _ = json.Marshal(&uf) gn.Put(ud) } c.Step("json marshal", func() { b, _ := json.Marshal(struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, }) w.Write(b) }) }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) put := false fixRead := false if time.Since(u.Read) > oldDuration { c.Warningf("u.Read too old, fixing: %v", u.Read) u.Read = time.Now().Add(-oldDuration) put = true fixRead = true c.Warningf("new: %v", u.Read) } read := make(Read) var uf Opml c.Step("unmarshal user data", func() { gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) now := time.Now() numStories := 0 c.Step("feed fetch + wait", func() { queue := make(chan *Feed) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var stories []*Story if u.Read.Before(f.Date) { fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories = make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } if f.Errors == 0 && f.NextUpdate.Before(now) { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {f.Url}, }) if _, err := taskqueue.Add(c, t, "update-manual"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } else { c.Warningf("manual feed update: %v", f.Url) } } f.Subscribe(c) lock.Lock() fl[f.Url] = stories numStories += len(stories) if len(stories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() } } for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) wg.Wait() }) if numStories > numStoriesLimit { c.Step("numStories", func() { c.Errorf("too many stories: %v", numStories) stories := make([]*Story, 0, numStories) for _, v := range fl { stories = append(stories, v...) } sort.Sort(sort.Reverse(Stories(stories))) last := stories[numStoriesLimit].Created stories = stories[:numStoriesLimit] u.Read = last put = true fixRead = true fl = make(map[string][]*Story) for _, s := range stories { fk := s.Parent.StringID() p := fl[fk] fl[fk] = append(p, s) } c.Errorf("filtered: %v, %v", len(stories), last) }) } if fixRead { c.Step("fix read", func() { nread := make(Read) for k, v := range fl { for _, s := range v { rs := readStory{Feed: k, Story: s.Id} if read[rs] { nread[rs] = true } } } c.Errorf("fix read: %v -> %v", len(read), len(nread)) read = nread var b bytes.Buffer gob.NewEncoder(&b).Encode(&read) ud.Read = b.Bytes() put = true }) } for k, v := range fl { newStories := make([]*Story, 0, len(v)) for _, s := range v { if !read[readStory{Feed: k, Story: s.Id}] { newStories = append(newStories, s) } } fl[k] = newStories } if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) put = true u.Read = last ud.Read = nil } } if updatedLinks { ud.Opml, _ = json.Marshal(&uf) put = true } if put { gn.PutMany(u, ud) } c.Step("json marshal", func() { o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := cleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} if err := gn.GetMulti([]interface{}{u, ud}); err != nil && !goon.NotFound(err, 1) { serveError(w, err) return } l := &Log{ Parent: ud.Parent, Id: time.Now().UnixNano(), Text: "list feeds", } l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml)) putU := false putUD := false fixRead := false if time.Since(u.Read) > oldDuration { u.Read = time.Now().Add(-oldDuration) putU = true fixRead = true l.Text += ", u.Read" } trialRemaining := 0 if STRIPE_KEY != "" && ud.Opml != nil { if u.Created.IsZero() { u.Created = time.Now() putU = true } else if time.Since(u.Created) > accountFreeDuration { b, _ := json.Marshal(struct { ErrorSubscription bool }{ true, }) w.Write(b) return } trialRemaining = int((accountFreeDuration-time.Since(u.Created))/time.Hour/24) + 1 } read := make(Read) var uf Opml c.Step("unmarshal user data", func(c mpg.Context) { gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func(c mpg.Context) { gn := goon.FromContext(appengine.Timeout(c, time.Minute)) for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()). Filter(IDX_COL+" >=", u.Read). KeysOnly(). Order("-" + IDX_COL). Limit(250) updatedLinks := false now := time.Now() numStories := 0 c.Step(fmt.Sprintf("feed unreads: %v", u.Read), func(c mpg.Context) { queue := make(chan *Feed) tc := make(chan *taskqueue.Task) done := make(chan bool) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { c.Step(f.Title, func(c mpg.Context) { defer wg.Done() var stories []*Story gn := goon.FromContext(appengine.Timeout(c, time.Minute)) if !f.Date.Before(u.Read) { fk := gn.Key(f) sq := q.Ancestor(fk) keys, _ := gn.GetAll(sq, nil) stories = make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) } if f.Link != opmlMap[f.Url].HtmlUrl { l.Text += fmt.Sprintf(", link: %v -> %v", opmlMap[f.Url].HtmlUrl, f.Link) updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } manualDone := false if time.Since(f.LastViewed) > time.Hour*24*2 { if f.NextUpdate.Equal(timeMax) { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, "last": {"1"}, }) manualDone = true } else { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-last"), url.Values{ "feed": {f.Url}, }) } } if !manualDone && now.Sub(f.NextUpdate) >= 0 { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, }) } lock.Lock() fl[f.Url] = stories numStories += len(stories) lock.Unlock() }) } } go taskSender(c, "update-manual", tc, done) for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) // wait for feeds to complete so there are no more tasks to queue wg.Wait() // then finish enqueuing tasks close(tc) <-done }) if numStories > 0 { c.Step("numStories", func(c mpg.Context) { stories := make([]*Story, 0, numStories) for _, v := range fl { stories = append(stories, v...) } sort.Sort(sort.Reverse(Stories(stories))) if len(stories) > numStoriesLimit { stories = stories[:numStoriesLimit] fl = make(map[string][]*Story) for _, s := range stories { fk := s.Parent.StringID() p := fl[fk] fl[fk] = append(p, s) } } last := stories[len(stories)-1].Created if u.Read.Before(last) { u.Read = last putU = true fixRead = true } }) } if fixRead { c.Step("fix read", func(c mpg.Context) { nread := make(Read) for k, v := range fl { for _, s := range v { rs := readStory{Feed: k, Story: s.Id} if read[rs] { nread[rs] = true } } } if len(nread) != len(read) { read = nread var b bytes.Buffer gob.NewEncoder(&b).Encode(&read) ud.Read = b.Bytes() putUD = true l.Text += ", fix read" } }) } numStories = 0 for k, v := range fl { newStories := make([]*Story, 0, len(v)) for _, s := range v { if !read[readStory{Feed: k, Story: s.Id}] { newStories = append(newStories, s) } } numStories += len(newStories) fl[k] = newStories } if numStories == 0 { l.Text += ", clear read" fixRead = false if ud.Read != nil { putUD = true ud.Read = nil } last := u.Read for _, v := range feeds { if last.Before(v.Date) { last = v.Date } } c.Infof("nothing here, move up: %v -> %v", u.Read, last) if u.Read.Before(last) { putU = true u.Read = last } } if updatedLinks { backupOPML(c) if o, err := json.Marshal(&uf); err == nil { ud.Opml = o putUD = true l.Text += ", update links" } else { c.Errorf("json UL err: %v, %v", err, uf) } } if putU { gn.Put(u) l.Text += ", putU" } if putUD { gn.Put(ud) l.Text += ", putUD" } l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml)) gn.Put(l) c.Step("json marshal", func(c mpg.Context) { gn := goon.FromContext(c) o := struct { Opml []*OpmlOutline Stories map[string][]*Story Options string TrialRemaining int Feeds []*Feed }{ Opml: uf.Outline, Stories: fl, Options: u.Options, TrialRemaining: trialRemaining, Feeds: feeds, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := sanitizer.CleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) read := make(Read) var uf Opml c.Step("unmarshal user data", func() { json.Unmarshal(ud.Read, &read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) now := time.Now() c.Step("feed fetch + wait", func() { queue := make(chan *Feed) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var newStories []*Story if u.Read.Before(f.Date) { c.Debugf("query for %v", f.Url) fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories := make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) for _, st := range stories { found := false for _, s := range read[f.Url] { if s == st.Id { found = true break } } if !found { newStories = append(newStories, st) } } } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } if f.Errors == 0 && f.NextUpdate.Before(now) { t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{ "feed": {f.Url}, }) if _, err := taskqueue.Add(c, t, "update-manual"); err != nil { c.Errorf("taskqueue error: %v", err.Error()) } else { c.Warningf("manual feed update: %v", f.Url) } } f.Subscribe(c) lock.Lock() fl[f.Url] = newStories if len(newStories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } lock.Unlock() } } for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) wg.Wait() }) if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) u.Read = last ud.Read = nil gn.PutMany(u, ud) } } if updatedLinks { ud.Opml, _ = json.Marshal(&uf) gn.Put(ud) } c.Step("json marshal", func() { o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, Options: u.Options, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := cleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) _ = utf8.RuneError }
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) { cu := user.Current(c) gn := goon.FromContext(c) u := &User{Id: cu.ID} ud := &UserData{Id: "data", Parent: gn.Key(u)} gn.GetMulti([]interface{}{u, ud}) putU := false putUD := false fixRead := false if time.Since(u.Read) > oldDuration { u.Read = time.Now().Add(-oldDuration) putU = true fixRead = true } read := make(Read) var uf Opml c.Step("unmarshal user data", func() { gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read) json.Unmarshal(ud.Opml, &uf) }) var feeds []*Feed opmlMap := make(map[string]*OpmlOutline) var merr error c.Step("fetch feeds", func() { for _, outline := range uf.Outline { if outline.XmlUrl == "" { for _, so := range outline.Outline { feeds = append(feeds, &Feed{Url: so.XmlUrl}) opmlMap[so.XmlUrl] = so } } else { feeds = append(feeds, &Feed{Url: outline.XmlUrl}) opmlMap[outline.XmlUrl] = outline } } merr = gn.GetMulti(feeds) }) lock := sync.Mutex{} fl := make(map[string][]*Story) q := datastore.NewQuery(gn.Key(&Story{}).Kind()) hasStories := false updatedLinks := false icons := make(map[string]string) noads := make(map[string]bool) now := time.Now() numStories := 0 c.Step("feed fetch + wait", func() { queue := make(chan *Feed) tc := make(chan *taskqueue.Task) wg := sync.WaitGroup{} feedProc := func() { for f := range queue { defer wg.Done() var stories []*Story if u.Read.Before(f.Date) { fk := gn.Key(f) sq := q.Ancestor(fk).Filter(IDX_COL+" >", u.Read).KeysOnly().Order("-" + IDX_COL) keys, _ := gn.GetAll(sq, nil) stories = make([]*Story, len(keys)) for j, key := range keys { stories[j] = &Story{ Id: key.StringID(), Parent: fk, } } gn.GetMulti(stories) } if f.Link != opmlMap[f.Url].HtmlUrl { updatedLinks = true opmlMap[f.Url].HtmlUrl = f.Link } manualDone := false if time.Since(f.LastViewed) > time.Hour*24*2 { if f.NextUpdate.Equal(timeMax) { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, "last": {"1"}, }) manualDone = true } else { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-last"), url.Values{ "feed": {f.Url}, }) } } if !manualDone && now.Sub(f.NextUpdate) >= 0 { tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{ "feed": {f.Url}, }) } lock.Lock() fl[f.Url] = stories numStories += len(stories) if len(stories) > 0 { hasStories = true } if f.Image != "" { icons[f.Url] = f.Image } if f.NoAds { noads[f.Url] = true } lock.Unlock() } } go func() { var tasks []*taskqueue.Task for t := range tc { tasks = append(tasks, t) if len(tasks) == 100 { taskqueue.AddMulti(c, tasks, "update-manual") c.Infof("added %v tasks", len(tasks)) tasks = tasks[0:0] } } if len(tasks) > 0 { taskqueue.AddMulti(c, tasks, "update-manual") c.Infof("added %v tasks", len(tasks)) } wg.Done() }() for i := 0; i < 20; i++ { go feedProc() } for i, f := range feeds { if goon.NotFound(merr, i) { continue } wg.Add(1) queue <- f } close(queue) // wait for feeds to complete so there are no more tasks to queue wg.Wait() wg.Add(1) // then finish enqueuing tasks close(tc) wg.Wait() }) if numStories > numStoriesLimit { c.Step("numStories", func() { c.Infof("too many stories: %v", numStories) stories := make([]*Story, 0, numStories) for _, v := range fl { stories = append(stories, v...) } sort.Sort(sort.Reverse(Stories(stories))) last := stories[numStoriesLimit].Created stories = stories[:numStoriesLimit] u.Read = last putU = true fixRead = true fl = make(map[string][]*Story) for _, s := range stories { fk := s.Parent.StringID() p := fl[fk] fl[fk] = append(p, s) } c.Infof("filtered: %v, %v", len(stories), last) }) } if fixRead { c.Step("fix read", func() { nread := make(Read) for k, v := range fl { for _, s := range v { rs := readStory{Feed: k, Story: s.Id} if read[rs] { nread[rs] = true } } } read = nread var b bytes.Buffer gob.NewEncoder(&b).Encode(&read) ud.Read = b.Bytes() putUD = true }) } for k, v := range fl { newStories := make([]*Story, 0, len(v)) for _, s := range v { if !read[readStory{Feed: k, Story: s.Id}] { newStories = append(newStories, s) } } fl[k] = newStories } if !hasStories { var last time.Time for _, f := range feeds { if last.Before(f.Date) { last = f.Date } } if u.Read.Before(last) { c.Debugf("setting %v read to %v", cu.ID, last) putU = true putUD = true u.Read = last ud.Read = nil } } if updatedLinks { backupOPML(c) if o, err := json.Marshal(&uf); err == nil { ud.Opml = o putUD = true } else { saveError(c, fmt.Sprintf("%v", uf), err) c.Errorf("json UL err: %v, %v", err, uf) } } if putU { gn.Put(u) } if putUD { gn.Put(ud) } c.Step("json marshal", func() { o := struct { Opml []*OpmlOutline Stories map[string][]*Story Icons map[string]string NoAds map[string]bool Options string }{ Opml: uf.Outline, Stories: fl, Icons: icons, NoAds: noads, Options: u.Options, } b, err := json.Marshal(o) if err != nil { c.Errorf("cleaning") for _, v := range fl { for _, s := range v { n := sanitizer.CleanNonUTF8(s.Summary) if n != s.Summary { s.Summary = n c.Errorf("cleaned %v", s.Id) gn.Put(s) } } } b, _ = json.Marshal(o) } w.Write(b) }) }