func persistStories(stories []stories.Story, db *database.Db) { count := 0 for _, story := range stories { // If the story doesn't already exist, persist it if !db.StoryExists(&story) { db.PersistStory(&story) count += 1 } } fmt.Printf("Saved %d new stories\n", count) }
func handleCrawls(cfg *config.Config, db *database.Db, broker *messaging.Broker, ticker *time.Ticker, quit chan os.Signal, wg *sync.WaitGroup) { defer wg.Done() for { select { case <-ticker.C: fmt.Printf("Enqueueing crawl requests ..\n") ids, err := db.FetchAllSourceIds() if err != nil { continue } for _, id := range ids { broker.EnqueueCrawl(int(id)) } fmt.Printf("Enqueueing crawl requests [done]\n") case <-quit: fmt.Printf("Shutting down crawling loop\n") ticker.Stop() return } } }