func main() {
feeds, err := hathor.GetFeeds()
if err != nil {
errlog <- err
}
fmt.Println("--- Feeds ---")
for key := range feeds {
fmt.Printf(" * %s\n", key)
go func(uri string, timeout int) {
feed := rss.New(timeout, true, chanHandler, itemHandler)
for {
if err := feed.Fetch(uri, nil); err != nil {
errlog <- fmt.Errorf("%s: %s", err, uri)
return
}
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}(feeds[key].Source, 5)
}
for {
select {
case res := <-log:
fmt.Println(res)
case res := <-errlog:
fmt.Printf("[e] %s\n", res)
}
}
}
func main() {
/*
resources := []Resource{
Resource{ Url: "http://zhihu.com/rss" },
Resource{ Url: "http://feed.zackyang.com/articles.xml" },
}
*/
for {
resources := getResources()
resources = []Resource{
Resource{Url: "http://zhihu.com/rss"},
Resource{Url: "http://feed.zackyang.com/articles.xml"},
}
// 获取所有订阅内容
for _, resource := range resources {
feed := rss.New(5, true, channelHandler, itemHandler)
if err := feed.Fetch(resource.Url, nil); err != nil {
fmt.Printf("Error: %s - %s", resource.Url, err)
}
}
<-time.After(time.Minute * 10)
}
}
func botPollSubscribe(bot tgbot.TgBot, msg tgbot.Message, uri string, timeout int, cr xmlx.CharsetFunc) {
// If already a rss, only add him to the db
// First, in cache:
id := fmt.Sprintf("%d", msg.Chat.ID)
ok := checkCache(uri, id)
if ok {
appendToCacheUsers(uri, msg.Chat.ID)
bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
return
}
ok = checkDb(uri, id)
if ok {
appendToCacheUsers(uri, msg.Chat.ID)
bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
return
}
firsttime := true
n_errors := 0
// Adding new RSS
feed := rss.New(timeout, true, chanHandler, botItemHandler(bot, firsttime))
for {
if err := feed.Fetch(uri, cr); err != nil {
// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
if !firsttime && n_errors < MAX_RETRIES {
fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
n_errors++
<-time.After(time.Duration(10) * time.Second)
continue
} else {
if firsttime {
bot.Answer(msg).Text(fmt.Sprintf("Bad RSS: %s, maybe the URL is bad.\nError msg: %s", uri, err.Error())).ReplyToMessage(msg.ID).End()
} else {
sendToAll(bot, uri, []NewStruct{
NewStruct{
Text: fmt.Sprintf("Hi! I had too many failures reading the RSS %s and I'm going to remove it, you can subscribe againg using:\n/sub %s", uri, uri),
},
})
}
fmt.Fprintf(os.Stderr, "Removing url %s\n", uri)
cleanBadUrl(uri)
return
}
}
n_errors = 0
if firsttime {
saveAllValues(uri, id)
appendToCacheUsers(uri, msg.Chat.ID)
setRssValue(buildKey("rss", uri, ""), "true")
firsttime = false
bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
}
// Every MINS_SLEEP mins
// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
}
}
func main() {
config, err := GetConfig()
if err != nil {
fmt.Printf("[e] %s\n", err)
return
}
for key := range config {
go func(uri string, timeout int) {
rssfeed := NewRssFeed(key, config[key])
feed := rss.New(timeout, true, rssfeed.Channels, rssfeed.Items)
for {
if err := feed.Fetch(uri, nil); err != nil {
fmt.Printf("[e] %s: %s\n", err, uri)
return
}
update := feed.SecondsTillUpdate()
fmt.Printf("%s - Updating again in %d seconds\n", key, update)
<-time.After(time.Duration(update) * time.Second)
}
}(config[key].Source, 5)
}
// Wait for episodes to arrive and download 'em
go ProcessEpisodes(download)
ServeFeeds()
}
// NewFetcher returns a Fetcher for uri.
func NewFetcher(uri string) Fetcher {
f := &fetcher{
uri: uri,
}
newChans := func(feed *rss.Feed, chans []*rss.Channel) {}
newItems := func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
for _, item := range items {
// Initialize guid to Atom Id
// item.Id is empty string if item comes from RSS feed
guid := item.Id
// if item comes from RSS feed Guid must be provided
if item.Guid != nil {
guid = *(item.Guid)
}
f.items = append(f.items, Item{
Channel: ch.Title,
Title: item.Title,
GUID: guid,
})
}
}
// min interval 1min, respect limits
f.feed = rss.New(1, true, newChans, newItems)
return f
}
// Return response
func (c *Postman) Run(user *commons.User, params []string) (string, error) {
feed := rss.New(10, false, nil, nil)
if err := feed.Fetch(c.cfg.Url, nil); err != nil {
return "", err
}
if len(feed.Channels) == 0 {
return "", errors.New("No items found in feed")
}
var response string
for _, item := range feed.Channels[0].Items {
if len(item.Links) == 0 {
continue
}
url := item.Links[0].Href
if _, ok := c.cache[url]; !ok {
response = fmt.Sprintf("%s: %s", item.Title, url)
c.cache[url] = 1
break
}
}
if response == "" {
return "", errors.New("You've read all news. Please update feed later.")
}
return response, nil
}
// PollFeed fetches the podcast feed at the given uri
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc) {
feed := rss.New(timeout, true, chanHandler, itemHandler)
if err := feed.Fetch(uri, cr); err != nil {
logger.Warning.Println("Feed parsing failure with "+uri, err)
return
}
//<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
func fetch(feed sourceFeed, master *feeds.Feed) {
fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
client := &http.Client{
Timeout: time.Second,
}
fetcher.FetchClient(feed.uri, client, nil)
}
func fetchFeed(uri string, timeout int, cr xmlx.CharsetFunc) (err error) {
feed := rss.New(timeout, true, chanHandler, itemHandler)
err = feed.Fetch(uri, cr)
if err != nil {
fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
return
}
return
}
func (fm *FeedMailer) fetch(url string) {
feed := rss.New(fm.prof.FetchTimeout, true, fm.chanHandler, fm.itemHandler)
for {
if err := feed.Fetch(url, nil); err != nil {
log.Println(err)
}
<-time.After(time.Duration(feed.SecondsTillUpdate()) * time.Second)
}
}
func readAllDbRss(bot tgbot.TgBot) {
allrss := loadFromDbPrefix("rss:")
// nmax := len(allrss)
blocks := 60.0 * float64(MINS_SLEEP) // Now every second in 9 mins. Every half second in a minute, 120 blocks
nseconds := float64(1) //60 / float64(blocks)
// module := int(math.Ceil(float64(nmax) / blocks))
i := 0
for urlkey := range allrss {
splitted := strings.Split(urlkey, ":")
if len(splitted) != 3 {
continue
}
uri := strings.Join(splitted[1:], ":")
j := i
i++
go func(uri string, firsttime bool) {
n_errors := 0
feed := rss.New(5, true, chanHandler, botItemHandler(bot, firsttime))
// Start calcule by groups, calculate how many sleep before execute
timeofsleep := math.Mod(float64(j), blocks) * nseconds // float64(int(j/module)) * nseconds
start := time.Now()
<-time.After(time.Duration(int(timeofsleep*1000)) * time.Millisecond)
fmt.Printf("%d (%s) started after %v seconds\n", j, uri, time.Since(start))
for {
if err := feed.Fetch(uri, charsetReader); err != nil {
// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
// if isAffordableNetworkError(err) && n_errors < MAX_RETRIES {
if n_errors < MAX_RETRIES {
fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
n_errors++
// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
continue
} else {
fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
fmt.Printf("Removing url %s\n", uri)
cleanBadUrl(uri)
return
}
}
n_errors = 0
if firsttime {
firsttime = false
setRssValue(urlkey, "true")
}
// Every MINS_SLEEP mins
// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
}
}(uri, true)
}
}
func fetch(url string) {
feed := feeder.New(60, false, handleChannel, handleItem)
err := feed.Fetch(url, nil)
if err != nil {
log.Println("error", url, err)
}
FinishedChannel <- url
}
func PollFeed(uri string, timeout int) {
feed := rss.New(timeout, true, nil, itemHandler)
for {
log.Print("Polling for new posts")
if err := feed.Fetch(uri, nil); err != nil {
log.Printf("Error fetching Feed : %s", err)
}
<-time.After(time.Minute * 5)
}
}
func PollFeed(wg *sync.WaitGroup, uri string, timeout int, cr xmlx.CharsetFunc) {
feed := rss.New(timeout, true, chanHandler, itemHandler)
// Decrement the counter when the goroutine completes.
defer wg.Done()
if err := feed.Fetch(uri, cr); err != nil {
log.Printf("[e] %s: %s", uri, err)
return
}
}
func main() {
feed := rss.New(5, true, chanHandler, itemHandler)
for {
refreshInterests()
if err := feed.Fetch("http://feeds.feedburner.com/TheFlightDeal", nil); err != nil {
fmt.Fprintf(os.Stderr, "[e] %s\n", err)
return
}
<-time.After(1 * time.Hour)
}
}
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc, out chan Record, limit int, source string) {
feed := rss.New(timeout, true, chanHandler, makeItemHandler(out, limit, source))
for {
if err := feed.Fetch(uri, cr); err != nil {
fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
return
}
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
func CreateFeeds() BuzzFeeds {
buzzFeedConfig = &GlobalConfig.BuzzFeed
var feeds BuzzFeeds
// initialize the buzzfeed RSS with a timeout of 5 seconds
for i := 0; i < len(buzzFeedConfig.Feeds); i++ {
feeds = append(feeds, &BuzzFeedRss{rss.New(timeout, true, feedChannelHandler, feedItemHandler)})
}
return feeds
}
func main() {
timeout := 5
uri := "http://live.nicovideo.jp/recent/rss?tab=live"
feed := rss.New(timeout, true, chanHandler, itemHandler)
err := feed.Fetch(uri, nil)
if err != nil {
fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
return
}
}
func PollFeed(uri string, timeout int) {
feed := rss.New(timeout, true, chanHandler, itemHandler)
for {
if err := feed.Fetch(uri); err != nil {
fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
return
}
<-time.After(feed.SecondsTillUpdate() * 1e9)
}
}
// PollFeed check for new content
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc) {
feed := rss.New(timeout, true, chanHandler, itemHandler)
for {
if err := feed.Fetch(uri, cr); err != nil {
fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
return
}
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
func scrapeRss(uri string, author string) {
feed := rss.New(RSS_TIMEOUT, true, chanHandler, customItemHandler(author))
for {
if err := feed.Fetch(uri, nil); err != nil {
fmt.Fprintf(os.Stderr, "[e.fetch] %s: %s", uri, err)
return
}
log.Printf("Sleeping for %d seconds on %s", feed.SecondsTillUpdate(), uri)
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
func (self *RssCrawler) PollFeed(timeout int) error {
feed := rss.New(timeout, true, self.chanHandler, self.itemHandler)
// for {
if err := feed.Fetch(self.Url, nil); err != nil {
goku.Logger().Errorf("%s: %s", self.Url, err)
return err
}
// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
// }
return nil
}
func botPollSubscribe(bot tgbot.TgBot, msg tgbot.Message, uri string, timeout int, cr xmlx.CharsetFunc) {
// If already a rss, only add him to the db
// First, in cache:
id := fmt.Sprintf("%d", msg.Chat.ID)
ok := checkCache(uri, id)
if ok {
appendToCacheUsers(uri, msg.Chat.ID)
bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
return
}
ok = checkDb(uri, id)
if ok {
appendToCacheUsers(uri, msg.Chat.ID)
bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
return
}
firsttime := true
n_errors := 0
// Adding new RSS
feed := rss.New(timeout, true, chanHandler, botItemHandler(bot, firsttime))
for {
if err := feed.Fetch(uri, cr); err != nil {
// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
if !firsttime && n_errors < MAX_RETRIES {
fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
n_errors++
<-time.After(time.Duration(10) * time.Second)
continue
} else {
if firsttime {
bot.Answer(msg).Text(fmt.Sprintf("Bad RSS: %s, maybe the URL is bad.\nError msg: %s", uri, err.Error())).ReplyToMessage(msg.ID).End()
}
return
}
}
n_errors = 0
if firsttime {
saveAllValues(uri, id)
appendToCacheUsers(uri, msg.Chat.ID)
setRssValue(buildKey("rss", uri, ""), "true")
firsttime = false
bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
}
// Usually every 5 mins
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
func getStories(feedUri string) {
timeout := 5 // 5 seconds
feedTarget := feedUri // rss feed to follow
feed := rss.New(timeout, true, nil, txNewItems)
for {
if err := feed.Fetch(feedTarget, nil); err != nil {
log.Printf("bot feed failure %s: %s", feedTarget, err)
continue
}
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
func initRss() {
chats := rc.SMembers("tgRssChats").Val()
for k := range chats {
feeds := rc.SMembers("tgRss:" + chats[k]).Val()
id, _ := strconv.Atoi(chats[k])
chat := &chat{id}
go func(feeds []string) {
for k := range feeds {
feed := rss.New(1, true, rssChan, chat.rssItem)
loopFeed(feed, feeds[k], chat.id)
}
}(feeds)
}
}
func getStories() error {
timeout := 5 // 5 seconds
feedTarget := feedUri // rss feed to follow
feed := rss.New(timeout, true, nil, txNewItems)
for {
if err := feed.Fetch(feedTarget, nil); err != nil {
log.Printf("bot feed failure %s: %s", feedTarget, err)
return err
}
lastCheck = time.Now()
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
return nil
}
func PollFeed(uri string) {
timeout := viper.GetInt("RSSTimeout")
if timeout < 1 {
timeout = 1
}
feed := rss.New(timeout, true, chanHandler, itemHandler)
for {
if err := feed.Fetch(uri, nil); err != nil {
fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
return
}
fmt.Printf("Sleeping for %d seconds on %s\n", feed.SecondsTillUpdate(), uri)
time.Sleep(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
// NewFetcher returns a Fetcher for uri.
func NewFetcher(uri string) Fetcher {
f := &fetcher{
uri: uri,
}
newChans := func(feed *rss.Feed, chans []*rss.Channel) {}
newItems := func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
for _, it := range items {
f.items = append(f.items, Item{
Channel: ch.Title,
GUID: it.Guid,
Title: it.Title,
})
}
}
f.feed = rss.New(1 /*minimum interval in minutes*/, true /*respect limit*/, newChans, newItems)
return f
}
func PollFeed(uri string, itemHandler rss.ItemHandler) {
feed := rss.New(timeout, true, chanHandler, itemHandler.ProcessItems)
for {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
err := feed.FetchClient(uri, client, nil)
if err != nil {
// don't die, just log and retry.
log.Printf("Error fetching %s: %s", uri, err)
} else {
log.Printf("Fetched %s\n", uri)
}
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
}
func fetchFeed(url string, timeout int) {
fmt.Printf("Fetching feed %s\n", url)
feed := rss.New(timeout, true, nil, itemHandler)
for {
if err := feed.Fetch(url, nil); err != nil {
fmt.Printf("%s: %s\n", url, err)
return
}
<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
wg.Done()
}