Пример #1
0
func reFetchFeeds() {
	logger.Info("[RF] Starting to re-fetching feeds")

	var articles []Article
	feedRegistry := NewFeedRegistry(config.Feeds)

	db.Where("refetch").Order("date desc").Limit(reFetchBatchSize).Find(&articles)

	for _, article := range articles {
		feed, err := feedRegistry.FindFeed(article.SourceSlug, article.CategorySlug)

		if err != nil {
			logger.Info("[RF] %s", err)
			continue
		}

		if err := FetchArticle(feed, &article); err != nil {
			logger.Info("[RF] Unable to fetch article: %s", err)
		}

		db.Model(&article).UpdateColumn("refetch", "false")
	}

	log.Info("[RF] Finished re-fetching feeds")
}
Пример #2
0
func fetchFeeds() {
	logger.Info("Starting to pull feeds")

	worker := NewBackgroundWorker(5)
	fetchingJob := func(item *FeedItem) { worker.Queue <- item.Fetch }

	feedRegistry := NewFeedRegistry(config.Feeds)
	feedRegistry.FetchFeeds(fetchingJob)

	worker.Process()
	log.Info("Finished pulling feeds")
}