Exemplo n.º 1
0
func doWork() (err error) {

	feed, err := rss.Fetch(BBC_FEED_URL)
	if err != nil {
		return err
	}

	for _, item := range feed.Items[:10] {
		if _, ok := visited[item.ID]; ok {
			continue
		}
		n := shruti.Notification{
			Title:        item.Title,
			Url:          item.Link,
			Key:          PROVIDER_NAME + item.ID,
			Priority:     shruti.PRIO_MED,
			Action:       shruti.ACT_PUSH,
			ProviderName: PROVIDER_NAME,
		}
		err = sClient.PushNotification(n)
		msg := "submitted"
		if err != nil {
			msg = err.Error()
		}
		visited[item.ID] = true
		log.Println(msg)
	}

	return nil
}
Exemplo n.º 2
0
// RSS抓取
func FetchRss(url string, category string) {
	feed, err := rss.Fetch(url)
	if err != nil {
		return
	}

	err = feed.Update()
	if err != nil {
		return
	}

	for k, _ := range feed.Items {
		words := strings.TrimSpace(feed.Items[k].Title)

		good, details, wordSlice := Scoring(words)

		result := &models.YuqingResult{}

		// 插入到结果表
		result.Id = words
		result.Category = category
		result.Good = good
		result.Detail = details
		result.WordSlice = wordSlice
		result.Insert()
	}
}
Exemplo n.º 3
0
Arquivo: main2.go Projeto: S010/test
func main() {
	urls := []string{
		"http://rss.slashdot.org/Slashdot/slashdotMain",
		"http://www.osnews.com/files/recent.xml",
	}

	feeds := make([]*rss.Feed, 0)

	for _, url := range urls {
		feed, err := rss.Fetch(url)
		if err != nil {
			fmt.Printf("%v: %v\n", url, err)
		}

		feeds = append(feeds, feed)

		err = feed.Update()
		if err != nil {
			fmt.Printf("%v: %v\n", feed.Link, err)
		}
	}

	fmt.Println("LATEST NEWS\n")
	t := template.Must(template.New("webpage").Parse(TEMPLATE))
	for _, feed := range feeds {
		err := t.Execute(os.Stdout, feed)
		if err != nil {
			fmt.Printf("error executing template: %v\n", err)
		}
	}
}
Exemplo n.º 4
0
Arquivo: api.go Projeto: henriwi/gorss
func asyncFetchFeeds(feeds []*rss.Feed) []*HttpResponse {
	ch := make(chan *HttpResponse)
	responses := []*HttpResponse{}

	for _, feed := range feeds {
		go func(f *rss.Feed) {
			fmt.Printf("Fetching %s\n", f.UpdateURL)
			feed, err := rss.Fetch(f.UpdateURL)

			if err != nil {
				fmt.Printf("Error in response %s. Using old feed.\n", err)
				ch <- &HttpResponse{f, err}
			} else {
				ch <- &HttpResponse{feed, err}
			}

		}(feed)
	}

	for {
		select {
		case r := <-ch:
			fmt.Printf("%s was fetched\n", r.feed.UpdateURL)
			responses = append(responses, r)
			if len(responses) == len(feeds) {
				return responses
			}
		case <-time.After(5 * time.Second):
			return responses
		}
	}
}
Exemplo n.º 5
0
// newfeedsplitter constructs a new feedsplitter
func newfeedsplitter(conn irc.SafeConn, url, name, color string, freq time.Duration, channels []string) (*feedspitter, error) {
	f, err := rss.Fetch(url)

	if err != nil {
		return nil, err
	}

	fs := &feedspitter{
		conn:     conn,
		feed:     f,
		name:     name,
		color:    color,
		freq:     freq,
		seen:     make(map[string]bool),
		channels: channels,
		stop:     make(chan bool),
	}

	// initial sweep of seen items
	for _, i := range fs.feed.Items {
		fs.seen[i.ID] = true
	}

	return fs, nil
}
Exemplo n.º 6
0
func getNewRedditPosts(feedUrl, source string) ([]*models.Post, error) {
	conn := models.CloneConnection()
	defer conn.Close()

	// get the feed
	feed, err := rss.Fetch(feedUrl)
	if err != nil {
		return []*models.Post{}, err
	}

	// iterate the feed and save the new items
	posts := make([]*models.Post, 0)
	for _, item := range feed.Items {
		exists, err := models.PostExists(conn, item.ID)
		if err != nil {
			return []*models.Post{}, err
		}

		if !exists {
			post := &models.Post{
				Title:       item.Title,
				Source:      source,
				Url:         item.Link,
				UniqueId:    item.ID,
				PublishedAt: item.Date,
			}

			posts = append(posts, post)
		}
	}

	return posts, nil
}
Exemplo n.º 7
0
func getFeed(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "feedName")

	feedSettings, err := config.LoadWidget(feedName)
	if err != nil {
		render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
		return
	}

	var items config.FeedItems

	feed, err := rss.Fetch(feedSettings.RssURL)
	if err != nil {
		render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
		return
	}

	for _, i := range feed.Items {
		items.ParseItem(i)
	}

	items, err = config.SaveFeed(feedName, items, feedSettings.Count)
	if err != nil {
		render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
		return
	}

	render.JSON(w, r, map[string]interface{}{"feed": items, "success": true})
}
Exemplo n.º 8
0
Arquivo: api.go Projeto: henriwi/gorss
func AddFeed(writer http.ResponseWriter, req *http.Request) {
	body, _ := ioutil.ReadAll(req.Body)
	var objmap map[string]string
	json.Unmarshal([]byte(body), &objmap)

	var url = objmap["url"]
	feed, err := rss.Fetch(url)

	if err != nil {
		fmt.Printf("Error fetching feed %s\n", err)
		writer.WriteHeader(http.StatusInternalServerError)
		return
	}

	err = db.Add(feed)

	if err != nil {
		fmt.Printf("%s\n", err)
		writer.WriteHeader(http.StatusBadRequest)
		return
	}

	writer.WriteHeader(http.StatusCreated)
	jsonResult, _ := json.Marshal(feed)
	fmt.Fprintf(writer, string(jsonResult))
}
Exemplo n.º 9
0
func init() {
	var err error
	news, err = rss.Fetch("http://ranssi.paivola.fi/rss.php")
	if err != nil {
		log.Errorf("[Posts] Failed to load front-page posts: %s", err)
	}
	lastupdate = util.Timestamp()
}
Exemplo n.º 10
0
func fetchFeed(url string, ch chan *rss.Feed) {
	fmt.Printf("Fetching %s \n", url)

	feed, err := rss.Fetch(url)

	if err != nil {
		fmt.Println("Failed to get feed %s\n", url)
	}

	ch <- feed
}
Exemplo n.º 11
0
func GetRss(entry Entry) (to_scrape []Target) {
	feed, err := rss.Fetch(entry.Url)
	if err != nil {
		fmt.Println("Entry:", entry.Name, "Error:", err)
	} else {
		for _, item := range feed.Items {
			scrape_item := Target{item.Link, item.Title, entry.Language, item.Date}
			to_scrape = append(to_scrape, scrape_item)
		}
	}
	return
}
Exemplo n.º 12
0
func checkFeed(rssURL string) (newEntryArr []*feedEl, err error) {

	if isFirstTime {
		isFirstTime = false
		_, err := rss.Fetch(rssURL)
		if err != nil {
			return nil, err
		}
	}
	log.Println("Checking for new rss items...")

	feed, err := rss.Fetch(rssURL)
	if err != nil {
		return nil, err
	}

	for _, e := range feed.Items {
		newEntryArr = append(newEntryArr,
			&feedEl{time: e.Date, title: e.Title, url: e.Link})
	}
	return
}
Exemplo n.º 13
0
func main() {
	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "Usage:\n\t%s [URL]\n", filepath.Base(os.Args[0]))
		os.Exit(2)
	}

	feed, err := rss.Fetch(os.Args[1])
	if err != nil {
		panic(err)
	}

	fmt.Printf("%+v\n", feed)
}
Exemplo n.º 14
0
func (c City) Search(category, query string) ([]Post, error) {
	feed, err := rss.Fetch(fmt.Sprintf(searchCityStr, c.URL, category, query))
	if err != nil {
		fmt.Println(fmt.Sprintf(searchCityStr, c.URL, category, query))
		return nil, err
	}

	results := make([]Post, 0)
	for _, item := range feed.Items {
		results = append(results, newPost(item))
	}
	return results, nil
}
Exemplo n.º 15
0
func (feed *Feed) Fetch() error {
	var err error
	logger.Info("Started fetching feed: %s", feed.Url)

	rss.CacheParsedItemIDs(false)
	feed.RawData, err = rss.Fetch(feed.Url)
	if err != nil {
		return err
	}

	logger.Info("Finished fetching feed: %s", feed.Url)
	logger.Info("There are %d items in %s", len(feed.RawData.Items), feed.Url)
	return nil
}
Exemplo n.º 16
0
Arquivo: feeds.go Projeto: heyLu/lp
func Fetch(fn string) (*feed.Feed, error) {
	fu, err := GetFeedUrl(fn)
	if err != nil {
		return nil, err
	}
	fn = fu

	f, err := feed.Fetch(fn)
	if err != nil {
		return nil, err
	}

	return f, nil
}
Exemplo n.º 17
0
// 获取最新动态
func GetChinaNew() {
	// 抓取rss
	feed, err := rss.Fetch(chinaRss)
	if err != nil {
		return
	}

	err = feed.Update()
	if err != nil {
		return
	}

	// 循环每一项rss数据
FindInItem:
	for k, _ := range feed.Items {
		sego := controllers.SegoController{}
		sentence := strings.TrimSpace(feed.Items[k].Title)
		content := strings.TrimSpace(feed.Items[k].Content)
		sentenceWords := sego.ToSlices(sentence, false)
		contentWords := sego.ToSlices(content, false)

		allWords := append(sentenceWords, contentWords...)

		// 循环每个分词
		for k, _ := range allWords {
			// 查找城市
			for m, _ := range citys {
				// 省份、城市名逐个搜索
				for p, _ := range citys[m] {
					if allWords[k] == citys[m][p] {
						// 情感分析
						good, details, wordSlice := controllers.Scoring(sentence)

						result := &models.YuqingResult{}

						// 插入到结果表
						result.Id = sentence
						result.Category = m
						result.Good = good
						result.Detail = details
						result.WordSlice = wordSlice
						result.Insert()

						continue FindInItem
					}
				}
			}
		}
	}
}
Exemplo n.º 18
0
func updateNews() {
	if news != nil {
		err := news.Update()
		if err != nil {
			log.Errorf("[Posts] Failed to update front-page posts: %s", err)
		}
	} else {
		var err error
		news, err = rss.Fetch("http://ranssi.paivola.fi/rss.php")
		if err != nil {
			log.Errorf("[Posts] Failed to update front-page posts: %s", err)
		}
	}
	lastupdate = util.Timestamp()
}
Exemplo n.º 19
0
func listenBuilds(lastMarker string, commandMsgs chan<- models.Message, connector models.Connector) (nextMarker string) {
	var displayOnStart = 0
	url := "https://" + connector.Login + ":" + connector.Pass + "@"
	url += connector.Server + "/builds/plugins/servlet/streams?local=true"
	feed, err := rss.Fetch(url)
	if err != nil {
		log.Print(err)
		return
	}
	for i := len(feed.Items) - 1; i >= 0; i-- {
		if connector.Debug {
			log.Print("Bamboo " + connector.ID + " item #" + strconv.Itoa(i) + " marker " + feed.Items[i].Date.String())
		}
		if lastMarker == "" {
			lastMarker = feed.Items[displayOnStart].Date.String()
		}
		item := feed.Items[i]
		if item.Date.String() > lastMarker {
			status := "NONE"
			if strings.Contains(item.Title, "successful") {
				status = "SUCCESS"
			}
			if strings.Contains(item.Title, "fail") {
				status = "FAIL"
			}
			var m models.Message
			m.In.ConnectorType = connector.Type
			m.In.ConnectorID = connector.ID
			m.In.Tags = connector.Tags
			m.In.Process = false
			m.Out.Text = "Bamboo Build " + html.UnescapeString(sanitize.HTML(item.Title))
			m.Out.Detail = html.UnescapeString(sanitize.HTML(item.Content))
			m.Out.Link = item.Link
			m.Out.Status = status
			commandMsgs <- m
			if i == 0 {
				lastMarker = item.Date.String()
			}
		}
	}
	nextMarker = lastMarker
	return nextMarker
}
Exemplo n.º 20
0
func callRss(lastMarker string, commandMsgs chan<- models.Message, connector models.Connector) (nextMarker string) {
	var displayOnStart = 0
	if connector.Debug {
		log.Print("Starting rss feed fetch for " + connector.ID)
	}
	feed, err := rss.Fetch(connector.Server)
	if err != nil {
		log.Print(err)
		return
	}
	if connector.Debug {
		log.Print("Feed count for " + connector.ID + ": " + strconv.Itoa(len(feed.Items)))
	}
	for i := len(feed.Items) - 1; i >= 0; i-- {
		if connector.Debug {
			log.Print("Feed " + connector.ID + " item #" + strconv.Itoa(i) + " marker " + feed.Items[i].Date.String())
		}
		if lastMarker == "" {
			lastMarker = feed.Items[displayOnStart].Date.String()
		}
		item := feed.Items[i]
		if item.Date.String() > lastMarker {
			var m models.Message
			m.In.ConnectorType = connector.Type
			m.In.ConnectorID = connector.ID
			m.In.Tags = connector.Tags
			m.In.Process = false
			m.Out.Text = connector.ID + " " + html.UnescapeString(sanitize.HTML(item.Title))
			m.Out.Detail = html.UnescapeString(sanitize.HTML(item.Content))
			m.Out.Link = item.Link
			commandMsgs <- m
			if i == 0 {
				lastMarker = item.Date.String()
			}
		}
	}
	nextMarker = lastMarker
	if connector.Debug {
		log.Print("Next marker for " + connector.ID + ": " + nextMarker)
	}
	return nextMarker
}
Exemplo n.º 21
0
func getFeed(urls <-chan string) <-chan *rss.Feed {
	out := make(chan *rss.Feed)

	go func() {
		for url := range urls {

			fmt.Printf("Fetching %s \n", url)

			feed, err := rss.Fetch(url)

			if err != nil {
				fmt.Printf("Failed to get feed %s\n", url)
			} else {
				out <- feed
			}
		}
		close(out)
	}()

	return out
}
Exemplo n.º 22
0
func main() {
	quit := make(chan os.Signal)
	signal.Notify(quit, os.Interrupt, os.Kill)

	feed, err := rss.Fetch("http://reddit.com/r/earthporn.rss")
	if err != nil {
		log.Fatal(err)
	}

	run(feed)

daLoop:
	for {
		select {
		case <-time.After(1 * time.Hour):
			feed.Update()
			run(feed)
		case <-quit:
			break daLoop
		}
	}
}
Exemplo n.º 23
0
func main() {
	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "Usage:\n\t%s [URL]\n", filepath.Base(os.Args[0]))
		os.Exit(2)
	}

	feed, err := rss.Fetch(os.Args[1])
	if err != nil {
		panic(err)
	}

	raw, err := json.Marshal(feed)
	if err != nil {
		panic(err)
	}

	buf := new(bytes.Buffer)
	if err := json.Indent(buf, raw, "", "\t"); err != nil {
		panic(err)
	}

	fmt.Println(buf.String())
}
Exemplo n.º 24
0
func addRss(m Message, ws *websocket.Conn) {
	feedUrl := getArgument(m, 3, ws)
	if feedUrl != "" {
		feedUrl = feedUrl[1 : len(feedUrl)-1]
		if govalidator.IsURL(feedUrl) {
			channelFeeds := feeds[m.Channel]
			for _, f := range channelFeeds {
				if f.Url == feedUrl {
					sendMessage(m.Channel, "Already tracking that url here, no worries homie :snowboarder:", ws)
					return
				}
			}

			var newFeed FeedBundle
			var err error
			newFeed.Url = feedUrl
			newFeed.RssHandler, err = rss.Fetch(feedUrl)
			if err != nil {
				sendMessage(m.Channel, "Incorrect url :OoOoOoOo", ws)
			} else {
				createNewFeedChecker := false
				if _, exists := feeds[m.Channel]; !exists {
					createNewFeedChecker = true
				}
				feeds[m.Channel] = append(feeds[m.Channel], newFeed)
				sendMessage(m.Channel, "Yes, sir. "+feedUrl+" added.", ws)

				if createNewFeedChecker {
					go checkFeed(m.Channel, feeds[m.Channel], ws)
				}
			}
		} else {
			sendMessage(m.Channel, "Incorrect url :OoOoOoOo", ws)
		}
	}
}
Exemplo n.º 25
0
func main() {
	// Connect to DB
	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}
	defer session.Close()

	session.SetMode(mgo.Monotonic, true)

	db := session.DB("buddio")

	avil := db.C("avilable_tracks")

	// Get RSS Feed
	//feed, err := rss.Fetch("http://feeds.5by5.tv/changelog")
	//feed, err := rss.Fetch("http://rss.cnn.com/services/podcasting/newscast/rss.xml")
	feed, err := rss.Fetch("http://feeds.kexp.org/kexp/songoftheday")
	if err != nil {
		panic(err)
	}

	for _, item := range feed.Items {
		err := avil.Insert(&Track{
			ID:     bson.NewObjectId(),
			Title:  item.Title,
			Length: parseLength(item.Length),
			Url:    item.Link,
			Type:   Music,
		})
		if err != nil {
			panic(err)
		}
		// fmt.Printf("%s - %d\n", item.Length, parseLength(item.Length))
	}

	// // Get google music tracks list
	// client := &http.Client{}

	// res, err := client.Do(req)
	// if err != nil {
	// 	panic(err)
	// }

	// defer res.Body.Close()

	// body, err := ioutil.ReadAll(res.Body)
	// if err != nil {
	// 	panic(err)
	// }

	// ts := &GTracks{}
	// err = json.Unmarshal(body, &ts)
	// if err != nil {
	// 	panic(err)
	// }

	// // Insert to DB
	// for _, gtrack := range ts.Data.Items {
	// 	t := &Track{
	// 		ID: gtrack.ID,
	// 		Title: gtrack.Artist + " - " + gtrack.Title,
	// 		Source: Source{
	// 			Url:
	// 		}
	// 	}
	// }
}
Exemplo n.º 26
0
func (f Feed) Update() (err error) {
	fmt.Printf("Updating feed %v\t%s\n", f.ID, f.Title)
	rssfeed, err := rss.Fetch(f.Url)
	if err != nil {
		glog.Errorf("rss.Fetch(%s): %s", f.Url, err)
		f.ErrorString = err.Error()
		_ = f.Save()
	}
	var guid_cache []string
	var new_items []Entry
	if rssfeed == nil {
		return err
	}
	if len(rssfeed.Items) < 1 {
		return err
	}

	for _, i := range rssfeed.Items {
		var e Entry
		e.Title = i.Title
		skippable, err := f.SkippableEntry(e)
		if err != nil {
			glog.Errorf("f.SkippableEntry(entry %s): %s", e.Title, err)
			skippable = false
		}
		if skippable {
			continue
		}
		fmt.Printf("%s", i)
		e.Content = template.HTML(html.EscapeString(i.Content))
		e.Date = fmt.Sprintf("%s", i.Date)
		e.FeedID = f.ID
		e.GUID = Escape_guid(i.Link)
		e.Link = i.Link
		e.Marked = "0"
		e.Unread = true
		guid_cache = append(guid_cache, e.GUID)
		new_items = append(new_items, e)
	}
	existing_entries := make(map[string]Entry)
	for _, e := range f.GetEntriesByParam(" guid in ('" + strings.Join(guid_cache, "', '") + "')") {
		existing_entries[e.GUID] = e
	}
	for _, e := range new_items {
		if _, ok := existing_entries[e.GUID]; ok {
			print(".")
			continue
		}
		skippable, err := f.SkippableEntry(e)
		if err != nil {
			glog.Errorf("f.SkippableEntry(e): %s", e, err)
			continue
		}
		if skippable {
			print(".")
			continue
		}
		print("+")
		e.Normalize()
		e.Save(f.UserName)
	}
	print("\n")
	f.ClearCache()
	return err
}
Exemplo n.º 27
0
func ReadFeed(task int, url string, resChan chan *rss.Feed) {
	log.Printf("(%03d) start fetch for %s", task, url)

	var feed *rss.Feed

	file := feedFile(url)

	// stat the file so we can tell if it exists
	_, err := os.Stat(file)
	if err != nil {
		log.Printf("(%03d) no cache file, fetching for first time", task)
		goto first_fetch
	} else {
		fd, err := os.Open(file)
		if err != nil {
			log.Printf("(%03d) could not read cache, refetching: %v", task, err)
			goto first_fetch
		} else {
			log.Printf("(%03d) reading feed cache from %v", task, file)
			dec := gob.NewDecoder(fd)
			err = dec.Decode(&feed)
			fd.Close()
			if err != nil {
				log.Printf("(%03d) invalid cache file, refetching: %v", task, err)
				goto first_fetch
			} else {
				err = feed.Update()
				if err != nil {
					log.Printf("(%03d) failed to update: %v", task, err)
					resChan <- nil
					return
				}
				goto loaded
			}
		}
	}

first_fetch:
	feed, err = rss.Fetch(url)
	if err != nil {
		log.Printf("(%03d) failed to fetch: %v", task, err)
		resChan <- nil
		return
	}

loaded:
	fd, err := os.Create(file)
	defer fd.Close()
	if err != nil {
		log.Println("(%03d) could not create cache file: %v", task, err)
	} else {
		enc := gob.NewEncoder(fd)
		err = enc.Encode(feed)
		if err != nil {
			log.Println("(%03d) could not write feed cache: %v", task, err)
		}
		log.Printf("(%03d) updated cache for %s", task, feed.Title)
	}

	resChan <- feed
}