func getNewRedditPosts() ([]*models.Post, error) {
conn := models.CloneConnection()
defer conn.Close()
// get the feed
feed, err := rss.Fetch(subredditUrl)
if err != nil {
return []*models.Post{}, err
}
// iterate the feed and save the new items
posts := make([]*models.Post, 0)
for _, item := range feed.Items {
exists, err := models.PostExists(conn, item.ID)
if err != nil {
return []*models.Post{}, err
}
if !exists {
post := &models.Post{
Title: item.Title,
Source: "reddit",
Url: item.Link,
UniqueId: item.ID,
PublishedAt: item.Date,
}
posts = append(posts, post)
}
}
return posts, nil
}
// getNewTopicsFromForum takes in a forum url name and returns up to 5 of the latest topics not saved in
// the database.
func getNewTopicsFromForum(forumSuffix string) ([]*models.Post, error) {
url := forumBaseUrl
if strings.Contains(forumBaseUrl, "%s") {
url = fmt.Sprintf(forumBaseUrl, forumSuffix)
}
conn := models.CloneConnection()
defer conn.Close()
doc, err := goquery.NewDocument(url)
if err != nil {
return []*models.Post{}, err
}
// find all of the topics
posts := make([]*models.Post, 0)
found := 0
doc.Find("tr[itemtype='http://schema.org/Article']").Each(func(i int, s *goquery.Selection) {
// we don't want any that are pinned
pinned := s.Find("span:contains('Pinned')")
if pinned.Size() == 0 {
found++
if found < 5 {
title := s.Find("span[itemprop='name']").Text()
url, _ := s.Find("a[itemprop='url']").Attr("href")
// stop now if we have already saved the topic
exists, err := models.PostExists(conn, url)
if err != nil || exists {
return
}
publishedAt, err := getDateFromPost(url)
if err != nil {
return
}
post := &models.Post{
Title: title,
Source: "forum",
Url: url,
UniqueId: url,
PublishedAt: publishedAt,
}
posts = append(posts, post)
}
}
})
return posts, nil
}