func (fm *FeedManager) stopUpdatingFeed(f content.Feed) { if f == nil { fm.logger.Infoln("No feed provided") return } data := f.Data() fm.logger.Infoln("Stopping feed update for " + data.Link) delete(fm.activeFeeds, data.Id) users := f.Users() if f.HasErr() { fm.logger.Printf("Error getting users for feed '%s': %v\n", f, f.Err()) } else { if len(users) == 0 { fm.logger.Infoln("Removing orphan feed " + f.String() + " from the database") for _, m := range fm.feedMonitors { if err := m.FeedDeleted(f); err != nil { fm.logger.Printf( "Error invoking monitor '%s' on deleted feed '%s': %v\n", reflect.TypeOf(m), f, err) } } f.Delete() if f.HasErr() { fm.logger.Printf("Error deleting feed '%s' from the repository: %v\n", f, f.Err()) } } } }
func (fm *FeedManager) requestFeedContent(f content.Feed) { if f == nil { fm.logger.Infoln("No feed provided") return } data := f.Data() fm.logger.Infoln("Requesting feed content for " + f.String()) resp, err := fm.client.Get(data.Link) if err != nil { data.UpdateError = err.Error() } else if resp.StatusCode != http.StatusOK { defer func() { // Drain the body so that the connection can be reused io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() }() data.UpdateError = httpStatusPrefix + strconv.Itoa(resp.StatusCode) } else { defer resp.Body.Close() data.UpdateError = "" buf := util.BufferPool.GetBuffer() defer util.BufferPool.Put(buf) if _, err := buf.ReadFrom(resp.Body); err == nil { hash := md5.Sum(buf.Bytes()) if b, ok := fm.lastUpdateHash[data.Id]; ok && bytes.Equal(b[:], hash[:]) { fm.logger.Infof("Content of feed %s is the same as the previous update\n", f) return } fm.lastUpdateHash[data.Id] = hash if pf, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil { f.Refresh(fm.processParserFeed(pf)) } else { data.UpdateError = err.Error() } } else { data.UpdateError = err.Error() } } if data.UpdateError != "" { fm.logger.Printf("Error updating feed '%s': %s\n", f, data.UpdateError) } f.Data(data) select { case <-fm.done: return default: fm.updateFeed(f) } }
func (fm FeedManager) processFeedUpdateMonitors(f content.Feed) { if len(f.NewArticles()) > 0 { for _, m := range fm.feedMonitors { if err := m.FeedUpdated(f); err != nil { fm.logger.Printf("Error invoking monitor '%s' on updated feed '%s': %v\n", reflect.TypeOf(m), f, err) } } } else { fm.logger.Infoln("No new articles for " + f.String()) } }
func (h Hubbub) subscription(s content.Subscription, f content.Feed, subscribe bool) { var err error fdata := f.Data() u := callbackURL(h.config, h.pattern, fdata.Id) body := url.Values{} body.Set("hub.callback", u) if subscribe { h.logger.Infoln("Subscribing to hubbub for " + f.String() + " with url " + u) body.Set("hub.mode", "subscribe") } else { h.logger.Infoln("Unsubscribing to hubbub for " + f.String() + " with url " + u) body.Set("hub.mode", "unsubscribe") } body.Set("hub.topic", fdata.Link) buf := util.BufferPool.GetBuffer() defer util.BufferPool.Put(buf) buf.WriteString(body.Encode()) req, _ := http.NewRequest("POST", s.Data().Link, buf) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("From", h.config.Hubbub.From) resp, err := h.client.Do(req) if err != nil { err = SubscriptionError{error: err, Subscription: s} } else if resp.StatusCode != 202 { err = SubscriptionError{error: errors.New("Expected response status 202, got " + resp.Status), Subscription: s} } if err == nil { if subscribe { h.subscribe <- s } else { h.unsubscribe <- s } } else { fdata.SubscribeError = err.Error() h.logger.Printf("Error subscribing to hub feed '%s': %s\n", f, err) f.Data(fdata) f.Update() if f.HasErr() { h.logger.Printf("Error updating feed database record for '%s': %s\n", f, f.Err()) } h.removeFeed <- f } }
func (fm *FeedManager) scoreFeedContent(f content.Feed) { if f == nil { fm.logger.Infoln("No feed provided") return } data := f.Data() if len(fm.config.Popularity.Providers) == 0 { fm.logger.Infoln("No popularity providers configured") return } if !fm.activeFeeds[data.Id] { fm.logger.Infof("Feed '%s' no longer active for scoring\n", f) return } fm.logger.Infoln("Scoring feed content for " + f.String()) articles := f.LatestArticles() if f.HasErr() { fm.logger.Printf("Error getting latest feed articles for '%s': %v\n", f, f.Err()) return } for i := range articles { sa := fm.repo.Article() sa.Data(articles[i].Data()) fm.scoreArticle <- sa } fm.logger.Infoln("Done scoring feed content for " + f.String()) select { case <-time.After(30 * time.Minute): go fm.scoreFeedContent(f) case <-fm.done: return } }