func (c Reddit) Feed(r string) revel.Result {
parts := strings.Split(r, ":")
r = parts[0]
// guid := "test"
// if len(parts) > 1 {
// guid = parts[1]
// }
const embedCacheFile = "embedCache"
embedCache, err := cache.LoadLRUS(100*1024, embedCacheFile)
if err != nil {
return HTML(fmt.Sprint(err))
}
embedder := embed.NewEmbedder(embedCache, map[string]string{
"EmbedlyAPIKey": "8b02b918d50e4e33b9152d62985d6241",
"maxWidth": "768",
})
feed, err := getRedditXMLFeed(r, embedder)
if err != nil {
return HTML("Cant get xml feed: " + fmt.Sprint(err))
}
// err = dedup.Dedup(feed, guid)
// if err != nil {
// return HTML("Cant dedup: " + fmt.Sprint(err))
// }
err = embedCache.Save(embedCacheFile)
if err != nil {
return HTML("Cant save embed cache: " + fmt.Sprint(err))
}
return c.RenderXml(feed)
}
func Dedup(feed *rss.Feed, guid string) (err error) {
// Hash item links, cache results.
const urlToHashCachePath = "dedup-hash-cache"
urlToHashCache, err := cache.LoadLRUS(100000, urlToHashCachePath)
if err != nil {
return fmt.Errorf("Cant load hash cache: %s", err)
}
hashes := hashItems(feed.Channel.Items, urlToHashCache)
err = urlToHashCache.Save(urlToHashCachePath)
if err != nil {
return fmt.Errorf("Cant safe hash cache: %s", err)
}
// Init served links hash caches.
allItemsCachePath := getAllItemsCachePath(guid)
allItemsCache, err := cache.LoadLRUS(100000, allItemsCachePath)
if err != nil {
return fmt.Errorf("Cant load items cache: %s", err)
}
recentItemsCachePath := getRecentItemsCachePath(guid, feed)
recentItemsCache, err := cache.LoadLRUS(100000, recentItemsCachePath)
if err != nil {
return fmt.Errorf("Cant load recent items cache: %s", err)
}
newItemsCache := cache.NewLRUS(100000)
// Prune duplicate links.
var rv []*rss.Item
for i, item := range feed.Channel.Items {
hash := hashes[i]
fmt.Print(item.Link + ": ")
if _, ok := newItemsCache.Get(hash); ok {
fmt.Println("found in new feed, duplicate")
// item.Title = "[In-Feed Repost] " + item.Title
continue
} else if _, ok := recentItemsCache.Get(hash); ok {
fmt.Println("found in last feed, new")
newItemsCache.Set(hash, "")
} else if _, ok := allItemsCache.Get(hash); ok {
fmt.Println("found in global feed, duplicate")
// item.Title = "[Repost] " + item.Title
continue
} else {
fmt.Println("not found in any feed, new")
allItemsCache.Set(hash, "")
newItemsCache.Set(hash, "")
}
rv = append(rv, item)
}
feed.Channel.Items = rv
// Save served links hash caches.
err = allItemsCache.Save(allItemsCachePath)
if err != nil {
return fmt.Errorf("Cant save all items cache: %s", err)
}
err = newItemsCache.Save(recentItemsCachePath)
if err != nil {
return fmt.Errorf("Cant save items cache: %s", err)
}
return
}
