func main() {
// Step 1: Establish connection to the message queue
// Step 2: Wait for a task to show up in the queue
// Step 3: Fetch the data for the given task, process it, generate stories
// Step 4: Put the data into a DB somewhere
// Step 5: Repeat
cfg := config.New()
broker := messaging.New(cfg)
for true {
fmt.Printf("Waiting for crawl request...\n")
sourceid := broker.DequeueCrawl()
db := database.New(cfg)
source := db.FetchSource(sourceid)
fmt.Printf("Fetching %s...\n", source.GetName())
articles := source.FetchNewData()
if articles != nil {
s := source.GenerateStories(articles)
persistStories(s, db)
} else {
fmt.Println("Yo something happened here\n")
}
db.PersistSource(source)
db.Close()
}
}
func main() {
cfg := config.New()
db := database.New(cfg)
broker := messaging.New(cfg)
quit := make(chan os.Signal, 1)
quit_crawl := make(chan os.Signal, 1)
quit_email := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt)
go func() {
for {
select {
case c := <-quit:
quit_crawl <- c
quit_email <- c
}
}
}()
wg := new(sync.WaitGroup)
// Every n minutes
num_minutes_crawl := 1 * time.Minute
ticker_crawl := time.NewTicker(num_minutes_crawl)
go handleCrawls(cfg, db, broker, ticker_crawl, quit_crawl, wg)
// Every n minutes
num_minutes_email := 24 * 60 * time.Minute
ticker_email := time.NewTicker(num_minutes_email)
go handleEmails(cfg, db, broker, ticker_email, quit_email, wg)
wg.Add(2)
wg.Wait()
// Cleanup
db.Close()
}
func main() {
fmt.Printf("This is the emailer\n")
cfg := config.New()
db := database.New(cfg)
nl := db.FetchNewsletter(1)
db.FetchNewsletterSources(nl)
nl.PrintNewsletter()
stories := db.FetchStoriesSince(nl.GetPubDate())
fmt.Printf("Original story count = %d\n", len(stories))
interesting_stories := nl.GetInterestingStories(stories)
fmt.Printf("Story count = %d\n", len(interesting_stories))
//nl.MarkPublished()
db.PersistNewsletter(nl)
html := producer.Produce(nl, interesting_stories)
// Uncomment these lines to enable sending emails
//cfg := config.New()
//send_email("*****@*****.**", nl.Title, html, cfg)
fmt.Printf("%s\n", html)
db.Close()
}