Beispiel #1
0
// handles scraped articles
// TODO: think about where this should be
func handleScrapedArticle(article scraper.Article) {
	if err := scraper.CheckFile(article.GetData()); err != nil {
		log.Warn("when checking article", article.GetTitle(), "got err:", err)
	}
	if err := storeArticle(article); err != nil {
		log.Error("failed to write article", article.GetTitle(), ":", err)
		return
	}
}
Beispiel #2
0
// write the provided article to the storage
func storeArticle(article scraper.Article) error {
	jsonStr, err := json.Marshal(article)
	if err != nil {
		return err
	}

	// take all spaces out of title
	// TODO: think about cleaning this up a little more
	fileName := strings.Replace(article.GetTitle(), " ", "", -1)
	path := "opinionatedData/" + fileName + ".json"

	err = ioutil.WriteFile(path, jsonStr, 0644)
	if err != nil {
		return err
	}

	log.Info("wrote article:", article.GetTitle(), "to location:", path)
	return nil
}