// handles scraped articles
// TODO: think about where this should be
func handleScrapedArticle(article scraper.Article) {
if err := scraper.CheckFile(article.GetData()); err != nil {
log.Warn("when checking article", article.GetTitle(), "got err:", err)
}
if err := storeArticle(article); err != nil {
log.Error("failed to write article", article.GetTitle(), ":", err)
return
}
}
// write the provided article to the storage
func storeArticle(article scraper.Article) error {
jsonStr, err := json.Marshal(article)
if err != nil {
return err
}
// take all spaces out of title
// TODO: think about cleaning this up a little more
fileName := strings.Replace(article.GetTitle(), " ", "", -1)
path := "opinionatedData/" + fileName + ".json"
err = ioutil.WriteFile(path, jsonStr, 0644)
if err != nil {
return err
}
log.Info("wrote article:", article.GetTitle(), "to location:", path)
return nil
}