Example #1
0
func persistStories(stories []stories.Story, db *database.Db) {
	count := 0
	for _, story := range stories {
		// If the story doesn't already exist, persist it
		if !db.StoryExists(&story) {
			db.PersistStory(&story)
			count += 1
		}
	}
	fmt.Printf("Saved %d new stories\n", count)
}
Example #2
0
func handleCrawls(cfg *config.Config, db *database.Db, broker *messaging.Broker, ticker *time.Ticker, quit chan os.Signal, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		select {
		case <-ticker.C:
			fmt.Printf("Enqueueing crawl requests ..\n")
			ids, err := db.FetchAllSourceIds()
			if err != nil {
				continue
			}
			for _, id := range ids {
				broker.EnqueueCrawl(int(id))
			}
			fmt.Printf("Enqueueing crawl requests [done]\n")

		case <-quit:
			fmt.Printf("Shutting down crawling loop\n")
			ticker.Stop()
			return
		}
	}
}