コード例 #1
0
ファイル: crawler.go プロジェクト: liquidmetal/broccoli-munch
func main() {
	// Step 1: Establish connection to the message queue
	// Step 2: Wait for a task to show up in the queue
	// Step 3: Fetch the data for the given task, process it, generate stories
	// Step 4: Put the data into a DB somewhere
	// Step 5: Repeat

	cfg := config.New()
	broker := messaging.New(cfg)

	for true {
		fmt.Printf("Waiting for crawl request...\n")
		sourceid := broker.DequeueCrawl()
		db := database.New(cfg)
		source := db.FetchSource(sourceid)
		fmt.Printf("Fetching %s...\n", source.GetName())

		articles := source.FetchNewData()

		if articles != nil {
			s := source.GenerateStories(articles)
			persistStories(s, db)
		} else {
			fmt.Println("Yo something happened here\n")
		}
		db.PersistSource(source)
		db.Close()
	}
}
コード例 #2
0
func main() {

	cfg := config.New()
	db := database.New(cfg)
	broker := messaging.New(cfg)

	quit := make(chan os.Signal, 1)
	quit_crawl := make(chan os.Signal, 1)
	quit_email := make(chan os.Signal, 1)

	signal.Notify(quit, os.Interrupt)
	go func() {
		for {
			select {
			case c := <-quit:
				quit_crawl <- c
				quit_email <- c
			}
		}
	}()

	wg := new(sync.WaitGroup)

	// Every n minutes
	num_minutes_crawl := 1 * time.Minute
	ticker_crawl := time.NewTicker(num_minutes_crawl)
	go handleCrawls(cfg, db, broker, ticker_crawl, quit_crawl, wg)

	// Every n minutes
	num_minutes_email := 24 * 60 * time.Minute
	ticker_email := time.NewTicker(num_minutes_email)
	go handleEmails(cfg, db, broker, ticker_email, quit_email, wg)
	wg.Add(2)

	wg.Wait()

	// Cleanup
	db.Close()
}
コード例 #3
0
func main() {
	cfg := config.New()
	broker := messaging.New(cfg)
	broker.EnqueueCrawl(3)
	fmt.Printf("Use this tool to control munch\n")
}