Beispiel #1
0
func main() {
	feeds, err := hathor.GetFeeds()
	if err != nil {
		errlog <- err
	}
	fmt.Println("--- Feeds ---")
	for key := range feeds {
		fmt.Printf(" * %s\n", key)

		go func(uri string, timeout int) {
			feed := rss.New(timeout, true, chanHandler, itemHandler)
			for {
				if err := feed.Fetch(uri, nil); err != nil {
					errlog <- fmt.Errorf("%s: %s", err, uri)
					return
				}
				<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
			}
		}(feeds[key].Source, 5)
	}

	for {
		select {
		case res := <-log:
			fmt.Println(res)
		case res := <-errlog:
			fmt.Printf("[e] %s\n", res)
		}
	}
}
Beispiel #2
0
func main() {
	/*
	  resources := []Resource{
	    Resource{ Url: "http://zhihu.com/rss" },
	    Resource{ Url: "http://feed.zackyang.com/articles.xml" },
	  }
	*/
	for {
		resources := getResources()
		resources = []Resource{
			Resource{Url: "http://zhihu.com/rss"},
			Resource{Url: "http://feed.zackyang.com/articles.xml"},
		}
		// 获取所有订阅内容
		for _, resource := range resources {
			feed := rss.New(5, true, channelHandler, itemHandler)
			if err := feed.Fetch(resource.Url, nil); err != nil {
				fmt.Printf("Error: %s - %s", resource.Url, err)
			}
		}

		<-time.After(time.Minute * 10)
	}

}
Beispiel #3
0
func botPollSubscribe(bot tgbot.TgBot, msg tgbot.Message, uri string, timeout int, cr xmlx.CharsetFunc) {
	// If already a rss, only add him to the db
	// First, in cache:
	id := fmt.Sprintf("%d", msg.Chat.ID)
	ok := checkCache(uri, id)
	if ok {
		appendToCacheUsers(uri, msg.Chat.ID)
		bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		return
	}
	ok = checkDb(uri, id)
	if ok {
		appendToCacheUsers(uri, msg.Chat.ID)
		bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		return
	}

	firsttime := true
	n_errors := 0
	// Adding new RSS
	feed := rss.New(timeout, true, chanHandler, botItemHandler(bot, firsttime))

	for {
		if err := feed.Fetch(uri, cr); err != nil {
			// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
			if !firsttime && n_errors < MAX_RETRIES {
				fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
				n_errors++
				<-time.After(time.Duration(10) * time.Second)
				continue
			} else {
				if firsttime {
					bot.Answer(msg).Text(fmt.Sprintf("Bad RSS: %s, maybe the URL is bad.\nError msg: %s", uri, err.Error())).ReplyToMessage(msg.ID).End()
				} else {
					sendToAll(bot, uri, []NewStruct{
						NewStruct{
							Text: fmt.Sprintf("Hi! I had too many failures reading the RSS %s and I'm going to remove it, you can subscribe againg using:\n/sub %s", uri, uri),
						},
					})
				}
				fmt.Fprintf(os.Stderr, "Removing url %s\n", uri)
				cleanBadUrl(uri)
				return
			}
		}

		n_errors = 0
		if firsttime {
			saveAllValues(uri, id)
			appendToCacheUsers(uri, msg.Chat.ID)
			setRssValue(buildKey("rss", uri, ""), "true")

			firsttime = false
			bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		}
		// Every MINS_SLEEP mins
		// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
		<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
	}
}
Beispiel #4
0
func main() {
	config, err := GetConfig()
	if err != nil {
		fmt.Printf("[e] %s\n", err)
		return
	}
	for key := range config {

		go func(uri string, timeout int) {
			rssfeed := NewRssFeed(key, config[key])
			feed := rss.New(timeout, true, rssfeed.Channels, rssfeed.Items)
			for {
				if err := feed.Fetch(uri, nil); err != nil {
					fmt.Printf("[e] %s: %s\n", err, uri)
					return
				}
				update := feed.SecondsTillUpdate()
				fmt.Printf("%s - Updating again in %d seconds\n", key, update)
				<-time.After(time.Duration(update) * time.Second)
			}
		}(config[key].Source, 5)
	}

	// Wait for episodes to arrive and download 'em
	go ProcessEpisodes(download)
	ServeFeeds()
}
Beispiel #5
0
// NewFetcher returns a Fetcher for uri.
func NewFetcher(uri string) Fetcher {
	f := &fetcher{
		uri: uri,
	}
	newChans := func(feed *rss.Feed, chans []*rss.Channel) {}
	newItems := func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for _, item := range items {
			// Initialize guid to Atom Id
			// item.Id is empty string if item comes from RSS feed
			guid := item.Id
			// if item comes from RSS feed Guid must be provided
			if item.Guid != nil {
				guid = *(item.Guid)
			}
			f.items = append(f.items, Item{
				Channel: ch.Title,
				Title:   item.Title,
				GUID:    guid,
			})
		}
	}
	// min interval 1min, respect limits
	f.feed = rss.New(1, true, newChans, newItems)
	return f
}
Beispiel #6
0
// Return response
func (c *Postman) Run(user *commons.User, params []string) (string, error) {
	feed := rss.New(10, false, nil, nil)

	if err := feed.Fetch(c.cfg.Url, nil); err != nil {
		return "", err
	}

	if len(feed.Channels) == 0 {
		return "", errors.New("No items found in feed")
	}

	var response string
	for _, item := range feed.Channels[0].Items {
		if len(item.Links) == 0 {
			continue
		}

		url := item.Links[0].Href
		if _, ok := c.cache[url]; !ok {
			response = fmt.Sprintf("%s: %s", item.Title, url)
			c.cache[url] = 1
			break
		}
	}

	if response == "" {
		return "", errors.New("You've read all news. Please update feed later.")
	}

	return response, nil
}
Beispiel #7
0
// PollFeed fetches the podcast feed at the given uri
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)
	if err := feed.Fetch(uri, cr); err != nil {
		logger.Warning.Println("Feed parsing failure with "+uri, err)
		return
	}
	//<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}
Beispiel #8
0
func fetch(feed sourceFeed, master *feeds.Feed) {
	fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
	client := &http.Client{
		Timeout: time.Second,
	}

	fetcher.FetchClient(feed.uri, client, nil)
}
Beispiel #9
0
func fetchFeed(uri string, timeout int, cr xmlx.CharsetFunc) (err error) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)
	err = feed.Fetch(uri, cr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
		return
	}
	return
}
Beispiel #10
0
func (fm *FeedMailer) fetch(url string) {
	feed := rss.New(fm.prof.FetchTimeout, true, fm.chanHandler, fm.itemHandler)
	for {
		if err := feed.Fetch(url, nil); err != nil {
			log.Println(err)
		}
		<-time.After(time.Duration(feed.SecondsTillUpdate()) * time.Second)
	}
}
Beispiel #11
0
func readAllDbRss(bot tgbot.TgBot) {
	allrss := loadFromDbPrefix("rss:")
	// nmax := len(allrss)
	blocks := 60.0 * float64(MINS_SLEEP) // Now every second in 9 mins. Every half second in a minute, 120 blocks
	nseconds := float64(1)               //60 / float64(blocks)
	// module := int(math.Ceil(float64(nmax) / blocks))
	i := 0
	for urlkey := range allrss {
		splitted := strings.Split(urlkey, ":")
		if len(splitted) != 3 {
			continue
		}
		uri := strings.Join(splitted[1:], ":")

		j := i
		i++
		go func(uri string, firsttime bool) {
			n_errors := 0
			feed := rss.New(5, true, chanHandler, botItemHandler(bot, firsttime))

			// Start calcule by groups, calculate how many sleep before execute
			timeofsleep := math.Mod(float64(j), blocks) * nseconds // float64(int(j/module)) * nseconds
			start := time.Now()
			<-time.After(time.Duration(int(timeofsleep*1000)) * time.Millisecond)
			fmt.Printf("%d (%s) started after %v seconds\n", j, uri, time.Since(start))

			for {
				if err := feed.Fetch(uri, charsetReader); err != nil {
					// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
					// if isAffordableNetworkError(err) && n_errors < MAX_RETRIES {
					if n_errors < MAX_RETRIES {
						fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
						n_errors++
						// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
						<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
						continue
					} else {
						fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
						fmt.Printf("Removing url %s\n", uri)
						cleanBadUrl(uri)
						return
					}
				}
				n_errors = 0

				if firsttime {
					firsttime = false
					setRssValue(urlkey, "true")
				}
				// Every MINS_SLEEP mins
				// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
				<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
			}
		}(uri, true)
	}
}
Beispiel #12
0
func fetch(url string) {
	feed := feeder.New(60, false, handleChannel, handleItem)
	err := feed.Fetch(url, nil)

	if err != nil {
		log.Println("error", url, err)
	}

	FinishedChannel <- url
}
Beispiel #13
0
func PollFeed(uri string, timeout int) {
	feed := rss.New(timeout, true, nil, itemHandler)
	for {
		log.Print("Polling for new posts")
		if err := feed.Fetch(uri, nil); err != nil {
			log.Printf("Error fetching Feed : %s", err)
		}
		<-time.After(time.Minute * 5)
	}
}
Beispiel #14
0
func PollFeed(wg *sync.WaitGroup, uri string, timeout int, cr xmlx.CharsetFunc) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)
	// Decrement the counter when the goroutine completes.
	defer wg.Done()

	if err := feed.Fetch(uri, cr); err != nil {
		log.Printf("[e] %s: %s", uri, err)
		return
	}

}
Beispiel #15
0
func main() {
	feed := rss.New(5, true, chanHandler, itemHandler)
	for {
		refreshInterests()
		if err := feed.Fetch("http://feeds.feedburner.com/TheFlightDeal", nil); err != nil {
			fmt.Fprintf(os.Stderr, "[e] %s\n", err)
			return
		}
		<-time.After(1 * time.Hour)
	}
}
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc, out chan Record, limit int, source string) {
	feed := rss.New(timeout, true, chanHandler, makeItemHandler(out, limit, source))

	for {
		if err := feed.Fetch(uri, cr); err != nil {
			fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
			return
		}

		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}
Beispiel #17
0
func CreateFeeds() BuzzFeeds {
	buzzFeedConfig = &GlobalConfig.BuzzFeed

	var feeds BuzzFeeds

	// initialize the buzzfeed RSS with a timeout of 5 seconds
	for i := 0; i < len(buzzFeedConfig.Feeds); i++ {
		feeds = append(feeds, &BuzzFeedRss{rss.New(timeout, true, feedChannelHandler, feedItemHandler)})
	}

	return feeds
}
Beispiel #18
0
func main() {
	timeout := 5
	uri := "http://live.nicovideo.jp/recent/rss?tab=live"

	feed := rss.New(timeout, true, chanHandler, itemHandler)
	err := feed.Fetch(uri, nil)

	if err != nil {
		fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
		return
	}
}
Beispiel #19
0
func PollFeed(uri string, timeout int) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)

	for {
		if err := feed.Fetch(uri); err != nil {
			fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
			return
		}

		<-time.After(feed.SecondsTillUpdate() * 1e9)
	}
}
Beispiel #20
0
// PollFeed check for new content
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)

	for {
		if err := feed.Fetch(uri, cr); err != nil {
			fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
			return
		}

		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}
Beispiel #21
0
func scrapeRss(uri string, author string) {
	feed := rss.New(RSS_TIMEOUT, true, chanHandler, customItemHandler(author))
	for {
		if err := feed.Fetch(uri, nil); err != nil {
			fmt.Fprintf(os.Stderr, "[e.fetch] %s: %s", uri, err)
			return
		}

		log.Printf("Sleeping for %d seconds on %s", feed.SecondsTillUpdate(), uri)
		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}
Beispiel #22
0
func (self *RssCrawler) PollFeed(timeout int) error {
	feed := rss.New(timeout, true, self.chanHandler, self.itemHandler)

	// for {
	if err := feed.Fetch(self.Url, nil); err != nil {
		goku.Logger().Errorf("%s: %s", self.Url, err)
		return err
	}

	// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	// }
	return nil
}
Beispiel #23
0
func botPollSubscribe(bot tgbot.TgBot, msg tgbot.Message, uri string, timeout int, cr xmlx.CharsetFunc) {
	// If already a rss, only add him to the db
	// First, in cache:
	id := fmt.Sprintf("%d", msg.Chat.ID)
	ok := checkCache(uri, id)
	if ok {
		appendToCacheUsers(uri, msg.Chat.ID)
		bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		return
	}
	ok = checkDb(uri, id)
	if ok {
		appendToCacheUsers(uri, msg.Chat.ID)
		bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		return
	}

	firsttime := true
	n_errors := 0
	// Adding new RSS
	feed := rss.New(timeout, true, chanHandler, botItemHandler(bot, firsttime))

	for {
		if err := feed.Fetch(uri, cr); err != nil {
			// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
			if !firsttime && n_errors < MAX_RETRIES {
				fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
				n_errors++
				<-time.After(time.Duration(10) * time.Second)
				continue
			} else {
				if firsttime {
					bot.Answer(msg).Text(fmt.Sprintf("Bad RSS: %s, maybe the URL is bad.\nError msg: %s", uri, err.Error())).ReplyToMessage(msg.ID).End()
				}
				return
			}
		}

		n_errors = 0
		if firsttime {
			saveAllValues(uri, id)
			appendToCacheUsers(uri, msg.Chat.ID)
			setRssValue(buildKey("rss", uri, ""), "true")

			firsttime = false
			bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		}
		// Usually every 5 mins
		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}
Beispiel #24
0
func getStories(feedUri string) {
	timeout := 5          // 5 seconds
	feedTarget := feedUri // rss feed to follow
	feed := rss.New(timeout, true, nil, txNewItems)
	for {
		if err := feed.Fetch(feedTarget, nil); err != nil {
			log.Printf("bot feed failure %s: %s", feedTarget, err)
			continue
		}

		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}

}
func initRss() {
	chats := rc.SMembers("tgRssChats").Val()
	for k := range chats {
		feeds := rc.SMembers("tgRss:" + chats[k]).Val()
		id, _ := strconv.Atoi(chats[k])
		chat := &chat{id}
		go func(feeds []string) {
			for k := range feeds {
				feed := rss.New(1, true, rssChan, chat.rssItem)
				loopFeed(feed, feeds[k], chat.id)
			}
		}(feeds)
	}
}
Beispiel #26
0
func getStories() error {
	timeout := 5          // 5 seconds
	feedTarget := feedUri // rss feed to follow
	feed := rss.New(timeout, true, nil, txNewItems)
	for {
		if err := feed.Fetch(feedTarget, nil); err != nil {
			log.Printf("bot feed failure %s: %s", feedTarget, err)
			return err
		}

		lastCheck = time.Now()
		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}

	return nil
}
Beispiel #27
0
func PollFeed(uri string) {
	timeout := viper.GetInt("RSSTimeout")
	if timeout < 1 {
		timeout = 1
	}
	feed := rss.New(timeout, true, chanHandler, itemHandler)

	for {
		if err := feed.Fetch(uri, nil); err != nil {
			fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
			return
		}
		fmt.Printf("Sleeping for %d seconds on %s\n", feed.SecondsTillUpdate(), uri)
		time.Sleep(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}
Beispiel #28
0
// NewFetcher returns a Fetcher for uri.
func NewFetcher(uri string) Fetcher {
	f := &fetcher{
		uri: uri,
	}
	newChans := func(feed *rss.Feed, chans []*rss.Channel) {}
	newItems := func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for _, it := range items {
			f.items = append(f.items, Item{
				Channel: ch.Title,
				GUID:    it.Guid,
				Title:   it.Title,
			})
		}
	}
	f.feed = rss.New(1 /*minimum interval in minutes*/, true /*respect limit*/, newChans, newItems)
	return f
}
Beispiel #29
0
func PollFeed(uri string, itemHandler rss.ItemHandler) {
	feed := rss.New(timeout, true, chanHandler, itemHandler.ProcessItems)

	for {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client := &http.Client{Transport: tr}
		err := feed.FetchClient(uri, client, nil)
		if err != nil {
			// don't die, just log and retry.
			log.Printf("Error fetching %s: %s", uri, err)
		} else {
			log.Printf("Fetched %s\n", uri)
		}

		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}
Beispiel #30
0
func fetchFeed(url string, timeout int) {

	fmt.Printf("Fetching feed %s\n", url)

	feed := rss.New(timeout, true, nil, itemHandler)

	for {

		if err := feed.Fetch(url, nil); err != nil {
			fmt.Printf("%s: %s\n", url, err)
			return
		}
		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))

	}

	wg.Done()

}