func main() {
// Step 1: Establish connection to the message queue
// Step 2: Wait for a task to show up in the queue
// Step 3: Fetch the data for the given task, process it, generate stories
// Step 4: Put the data into a DB somewhere
// Step 5: Repeat
cfg := config.New()
broker := messaging.New(cfg)
for true {
fmt.Printf("Waiting for crawl request...\n")
sourceid := broker.DequeueCrawl()
db := database.New(cfg)
source := db.FetchSource(sourceid)
fmt.Printf("Fetching %s...\n", source.GetName())
articles := source.FetchNewData()
if articles != nil {
s := source.GenerateStories(articles)
persistStories(s, db)
} else {
fmt.Println("Yo something happened here\n")
}
db.PersistSource(source)
db.Close()
}
}
func main() {
cfg := config.New()
db := database.New(cfg)
broker := messaging.New(cfg)
quit := make(chan os.Signal, 1)
quit_crawl := make(chan os.Signal, 1)
quit_email := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt)
go func() {
for {
select {
case c := <-quit:
quit_crawl <- c
quit_email <- c
}
}
}()
wg := new(sync.WaitGroup)
// Every n minutes
num_minutes_crawl := 1 * time.Minute
ticker_crawl := time.NewTicker(num_minutes_crawl)
go handleCrawls(cfg, db, broker, ticker_crawl, quit_crawl, wg)
// Every n minutes
num_minutes_email := 24 * 60 * time.Minute
ticker_email := time.NewTicker(num_minutes_email)
go handleEmails(cfg, db, broker, ticker_email, quit_email, wg)
wg.Add(2)
wg.Wait()
// Cleanup
db.Close()
}
func main() {
cfg := config.New()
broker := messaging.New(cfg)
broker.EnqueueCrawl(3)
fmt.Printf("Use this tool to control munch\n")
}