示例#1
0
文件: reddit.go 项目: pavelb/gorss
func (c Reddit) Feed(r string) revel.Result {
	parts := strings.Split(r, ":")
	r = parts[0]
	// guid := "test"
	// if len(parts) > 1 {
	// 	guid = parts[1]
	// }

	const embedCacheFile = "embedCache"

	embedCache, err := cache.LoadLRUS(100*1024, embedCacheFile)
	if err != nil {
		return HTML(fmt.Sprint(err))
	}
	embedder := embed.NewEmbedder(embedCache, map[string]string{
		"EmbedlyAPIKey": "8b02b918d50e4e33b9152d62985d6241",
		"maxWidth":      "768",
	})

	feed, err := getRedditXMLFeed(r, embedder)
	if err != nil {
		return HTML("Cant get xml feed: " + fmt.Sprint(err))
	}

	// err = dedup.Dedup(feed, guid)
	// if err != nil {
	// 	return HTML("Cant dedup: " + fmt.Sprint(err))
	// }

	err = embedCache.Save(embedCacheFile)
	if err != nil {
		return HTML("Cant save embed cache: " + fmt.Sprint(err))
	}

	return c.RenderXml(feed)
}
示例#2
0
文件: dedup.go 项目: pavelb/gorss
func Dedup(feed *rss.Feed, guid string) (err error) {
	// Hash item links, cache results.
	const urlToHashCachePath = "dedup-hash-cache"
	urlToHashCache, err := cache.LoadLRUS(100000, urlToHashCachePath)
	if err != nil {
		return fmt.Errorf("Cant load hash cache: %s", err)
	}
	hashes := hashItems(feed.Channel.Items, urlToHashCache)
	err = urlToHashCache.Save(urlToHashCachePath)
	if err != nil {
		return fmt.Errorf("Cant safe hash cache: %s", err)
	}

	// Init served links hash caches.
	allItemsCachePath := getAllItemsCachePath(guid)
	allItemsCache, err := cache.LoadLRUS(100000, allItemsCachePath)
	if err != nil {
		return fmt.Errorf("Cant load items cache: %s", err)
	}
	recentItemsCachePath := getRecentItemsCachePath(guid, feed)
	recentItemsCache, err := cache.LoadLRUS(100000, recentItemsCachePath)
	if err != nil {
		return fmt.Errorf("Cant load recent items cache: %s", err)
	}
	newItemsCache := cache.NewLRUS(100000)

	// Prune duplicate links.
	var rv []*rss.Item
	for i, item := range feed.Channel.Items {
		hash := hashes[i]

		fmt.Print(item.Link + ": ")

		if _, ok := newItemsCache.Get(hash); ok {
			fmt.Println("found in new feed, duplicate")
			// item.Title = "[In-Feed Repost] " + item.Title
			continue
		} else if _, ok := recentItemsCache.Get(hash); ok {
			fmt.Println("found in last feed, new")
			newItemsCache.Set(hash, "")
		} else if _, ok := allItemsCache.Get(hash); ok {
			fmt.Println("found in global feed, duplicate")
			// item.Title = "[Repost] " + item.Title
			continue
		} else {
			fmt.Println("not found in any feed, new")
			allItemsCache.Set(hash, "")
			newItemsCache.Set(hash, "")
		}

		rv = append(rv, item)
	}
	feed.Channel.Items = rv

	// Save served links hash caches.
	err = allItemsCache.Save(allItemsCachePath)
	if err != nil {
		return fmt.Errorf("Cant save all items cache: %s", err)
	}
	err = newItemsCache.Save(recentItemsCachePath)
	if err != nil {
		return fmt.Errorf("Cant save items cache: %s", err)
	}
	return
}