func doWork() (err error) {
feed, err := rss.Fetch(BBC_FEED_URL)
if err != nil {
return err
}
for _, item := range feed.Items[:10] {
if _, ok := visited[item.ID]; ok {
continue
}
n := shruti.Notification{
Title: item.Title,
Url: item.Link,
Key: PROVIDER_NAME + item.ID,
Priority: shruti.PRIO_MED,
Action: shruti.ACT_PUSH,
ProviderName: PROVIDER_NAME,
}
err = sClient.PushNotification(n)
msg := "submitted"
if err != nil {
msg = err.Error()
}
visited[item.ID] = true
log.Println(msg)
}
return nil
}
// RSS抓取
func FetchRss(url string, category string) {
feed, err := rss.Fetch(url)
if err != nil {
return
}
err = feed.Update()
if err != nil {
return
}
for k, _ := range feed.Items {
words := strings.TrimSpace(feed.Items[k].Title)
good, details, wordSlice := Scoring(words)
result := &models.YuqingResult{}
// 插入到结果表
result.Id = words
result.Category = category
result.Good = good
result.Detail = details
result.WordSlice = wordSlice
result.Insert()
}
}
func main() {
urls := []string{
"http://rss.slashdot.org/Slashdot/slashdotMain",
"http://www.osnews.com/files/recent.xml",
}
feeds := make([]*rss.Feed, 0)
for _, url := range urls {
feed, err := rss.Fetch(url)
if err != nil {
fmt.Printf("%v: %v\n", url, err)
}
feeds = append(feeds, feed)
err = feed.Update()
if err != nil {
fmt.Printf("%v: %v\n", feed.Link, err)
}
}
fmt.Println("LATEST NEWS\n")
t := template.Must(template.New("webpage").Parse(TEMPLATE))
for _, feed := range feeds {
err := t.Execute(os.Stdout, feed)
if err != nil {
fmt.Printf("error executing template: %v\n", err)
}
}
}
func asyncFetchFeeds(feeds []*rss.Feed) []*HttpResponse {
ch := make(chan *HttpResponse)
responses := []*HttpResponse{}
for _, feed := range feeds {
go func(f *rss.Feed) {
fmt.Printf("Fetching %s\n", f.UpdateURL)
feed, err := rss.Fetch(f.UpdateURL)
if err != nil {
fmt.Printf("Error in response %s. Using old feed.\n", err)
ch <- &HttpResponse{f, err}
} else {
ch <- &HttpResponse{feed, err}
}
}(feed)
}
for {
select {
case r := <-ch:
fmt.Printf("%s was fetched\n", r.feed.UpdateURL)
responses = append(responses, r)
if len(responses) == len(feeds) {
return responses
}
case <-time.After(5 * time.Second):
return responses
}
}
}
// newfeedsplitter constructs a new feedsplitter
func newfeedsplitter(conn irc.SafeConn, url, name, color string, freq time.Duration, channels []string) (*feedspitter, error) {
f, err := rss.Fetch(url)
if err != nil {
return nil, err
}
fs := &feedspitter{
conn: conn,
feed: f,
name: name,
color: color,
freq: freq,
seen: make(map[string]bool),
channels: channels,
stop: make(chan bool),
}
// initial sweep of seen items
for _, i := range fs.feed.Items {
fs.seen[i.ID] = true
}
return fs, nil
}
func getNewRedditPosts(feedUrl, source string) ([]*models.Post, error) {
conn := models.CloneConnection()
defer conn.Close()
// get the feed
feed, err := rss.Fetch(feedUrl)
if err != nil {
return []*models.Post{}, err
}
// iterate the feed and save the new items
posts := make([]*models.Post, 0)
for _, item := range feed.Items {
exists, err := models.PostExists(conn, item.ID)
if err != nil {
return []*models.Post{}, err
}
if !exists {
post := &models.Post{
Title: item.Title,
Source: source,
Url: item.Link,
UniqueId: item.ID,
PublishedAt: item.Date,
}
posts = append(posts, post)
}
}
return posts, nil
}
func getFeed(w http.ResponseWriter, r *http.Request) {
feedName := chi.URLParam(r, "feedName")
feedSettings, err := config.LoadWidget(feedName)
if err != nil {
render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
return
}
var items config.FeedItems
feed, err := rss.Fetch(feedSettings.RssURL)
if err != nil {
render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
return
}
for _, i := range feed.Items {
items.ParseItem(i)
}
items, err = config.SaveFeed(feedName, items, feedSettings.Count)
if err != nil {
render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
return
}
render.JSON(w, r, map[string]interface{}{"feed": items, "success": true})
}
func AddFeed(writer http.ResponseWriter, req *http.Request) {
body, _ := ioutil.ReadAll(req.Body)
var objmap map[string]string
json.Unmarshal([]byte(body), &objmap)
var url = objmap["url"]
feed, err := rss.Fetch(url)
if err != nil {
fmt.Printf("Error fetching feed %s\n", err)
writer.WriteHeader(http.StatusInternalServerError)
return
}
err = db.Add(feed)
if err != nil {
fmt.Printf("%s\n", err)
writer.WriteHeader(http.StatusBadRequest)
return
}
writer.WriteHeader(http.StatusCreated)
jsonResult, _ := json.Marshal(feed)
fmt.Fprintf(writer, string(jsonResult))
}
func init() {
var err error
news, err = rss.Fetch("http://ranssi.paivola.fi/rss.php")
if err != nil {
log.Errorf("[Posts] Failed to load front-page posts: %s", err)
}
lastupdate = util.Timestamp()
}
func fetchFeed(url string, ch chan *rss.Feed) {
fmt.Printf("Fetching %s \n", url)
feed, err := rss.Fetch(url)
if err != nil {
fmt.Println("Failed to get feed %s\n", url)
}
ch <- feed
}
func GetRss(entry Entry) (to_scrape []Target) {
feed, err := rss.Fetch(entry.Url)
if err != nil {
fmt.Println("Entry:", entry.Name, "Error:", err)
} else {
for _, item := range feed.Items {
scrape_item := Target{item.Link, item.Title, entry.Language, item.Date}
to_scrape = append(to_scrape, scrape_item)
}
}
return
}
func checkFeed(rssURL string) (newEntryArr []*feedEl, err error) {
if isFirstTime {
isFirstTime = false
_, err := rss.Fetch(rssURL)
if err != nil {
return nil, err
}
}
log.Println("Checking for new rss items...")
feed, err := rss.Fetch(rssURL)
if err != nil {
return nil, err
}
for _, e := range feed.Items {
newEntryArr = append(newEntryArr,
&feedEl{time: e.Date, title: e.Title, url: e.Link})
}
return
}
func main() {
if len(os.Args) == 1 {
fmt.Fprintf(os.Stderr, "Usage:\n\t%s [URL]\n", filepath.Base(os.Args[0]))
os.Exit(2)
}
feed, err := rss.Fetch(os.Args[1])
if err != nil {
panic(err)
}
fmt.Printf("%+v\n", feed)
}
func (c City) Search(category, query string) ([]Post, error) {
feed, err := rss.Fetch(fmt.Sprintf(searchCityStr, c.URL, category, query))
if err != nil {
fmt.Println(fmt.Sprintf(searchCityStr, c.URL, category, query))
return nil, err
}
results := make([]Post, 0)
for _, item := range feed.Items {
results = append(results, newPost(item))
}
return results, nil
}
func (feed *Feed) Fetch() error {
var err error
logger.Info("Started fetching feed: %s", feed.Url)
rss.CacheParsedItemIDs(false)
feed.RawData, err = rss.Fetch(feed.Url)
if err != nil {
return err
}
logger.Info("Finished fetching feed: %s", feed.Url)
logger.Info("There are %d items in %s", len(feed.RawData.Items), feed.Url)
return nil
}
func Fetch(fn string) (*feed.Feed, error) {
fu, err := GetFeedUrl(fn)
if err != nil {
return nil, err
}
fn = fu
f, err := feed.Fetch(fn)
if err != nil {
return nil, err
}
return f, nil
}
// 获取最新动态
func GetChinaNew() {
// 抓取rss
feed, err := rss.Fetch(chinaRss)
if err != nil {
return
}
err = feed.Update()
if err != nil {
return
}
// 循环每一项rss数据
FindInItem:
for k, _ := range feed.Items {
sego := controllers.SegoController{}
sentence := strings.TrimSpace(feed.Items[k].Title)
content := strings.TrimSpace(feed.Items[k].Content)
sentenceWords := sego.ToSlices(sentence, false)
contentWords := sego.ToSlices(content, false)
allWords := append(sentenceWords, contentWords...)
// 循环每个分词
for k, _ := range allWords {
// 查找城市
for m, _ := range citys {
// 省份、城市名逐个搜索
for p, _ := range citys[m] {
if allWords[k] == citys[m][p] {
// 情感分析
good, details, wordSlice := controllers.Scoring(sentence)
result := &models.YuqingResult{}
// 插入到结果表
result.Id = sentence
result.Category = m
result.Good = good
result.Detail = details
result.WordSlice = wordSlice
result.Insert()
continue FindInItem
}
}
}
}
}
}
func updateNews() {
if news != nil {
err := news.Update()
if err != nil {
log.Errorf("[Posts] Failed to update front-page posts: %s", err)
}
} else {
var err error
news, err = rss.Fetch("http://ranssi.paivola.fi/rss.php")
if err != nil {
log.Errorf("[Posts] Failed to update front-page posts: %s", err)
}
}
lastupdate = util.Timestamp()
}
func listenBuilds(lastMarker string, commandMsgs chan<- models.Message, connector models.Connector) (nextMarker string) {
var displayOnStart = 0
url := "https://" + connector.Login + ":" + connector.Pass + "@"
url += connector.Server + "/builds/plugins/servlet/streams?local=true"
feed, err := rss.Fetch(url)
if err != nil {
log.Print(err)
return
}
for i := len(feed.Items) - 1; i >= 0; i-- {
if connector.Debug {
log.Print("Bamboo " + connector.ID + " item #" + strconv.Itoa(i) + " marker " + feed.Items[i].Date.String())
}
if lastMarker == "" {
lastMarker = feed.Items[displayOnStart].Date.String()
}
item := feed.Items[i]
if item.Date.String() > lastMarker {
status := "NONE"
if strings.Contains(item.Title, "successful") {
status = "SUCCESS"
}
if strings.Contains(item.Title, "fail") {
status = "FAIL"
}
var m models.Message
m.In.ConnectorType = connector.Type
m.In.ConnectorID = connector.ID
m.In.Tags = connector.Tags
m.In.Process = false
m.Out.Text = "Bamboo Build " + html.UnescapeString(sanitize.HTML(item.Title))
m.Out.Detail = html.UnescapeString(sanitize.HTML(item.Content))
m.Out.Link = item.Link
m.Out.Status = status
commandMsgs <- m
if i == 0 {
lastMarker = item.Date.String()
}
}
}
nextMarker = lastMarker
return nextMarker
}
func callRss(lastMarker string, commandMsgs chan<- models.Message, connector models.Connector) (nextMarker string) {
var displayOnStart = 0
if connector.Debug {
log.Print("Starting rss feed fetch for " + connector.ID)
}
feed, err := rss.Fetch(connector.Server)
if err != nil {
log.Print(err)
return
}
if connector.Debug {
log.Print("Feed count for " + connector.ID + ": " + strconv.Itoa(len(feed.Items)))
}
for i := len(feed.Items) - 1; i >= 0; i-- {
if connector.Debug {
log.Print("Feed " + connector.ID + " item #" + strconv.Itoa(i) + " marker " + feed.Items[i].Date.String())
}
if lastMarker == "" {
lastMarker = feed.Items[displayOnStart].Date.String()
}
item := feed.Items[i]
if item.Date.String() > lastMarker {
var m models.Message
m.In.ConnectorType = connector.Type
m.In.ConnectorID = connector.ID
m.In.Tags = connector.Tags
m.In.Process = false
m.Out.Text = connector.ID + " " + html.UnescapeString(sanitize.HTML(item.Title))
m.Out.Detail = html.UnescapeString(sanitize.HTML(item.Content))
m.Out.Link = item.Link
commandMsgs <- m
if i == 0 {
lastMarker = item.Date.String()
}
}
}
nextMarker = lastMarker
if connector.Debug {
log.Print("Next marker for " + connector.ID + ": " + nextMarker)
}
return nextMarker
}
func getFeed(urls <-chan string) <-chan *rss.Feed {
out := make(chan *rss.Feed)
go func() {
for url := range urls {
fmt.Printf("Fetching %s \n", url)
feed, err := rss.Fetch(url)
if err != nil {
fmt.Printf("Failed to get feed %s\n", url)
} else {
out <- feed
}
}
close(out)
}()
return out
}
func main() {
quit := make(chan os.Signal)
signal.Notify(quit, os.Interrupt, os.Kill)
feed, err := rss.Fetch("http://reddit.com/r/earthporn.rss")
if err != nil {
log.Fatal(err)
}
run(feed)
daLoop:
for {
select {
case <-time.After(1 * time.Hour):
feed.Update()
run(feed)
case <-quit:
break daLoop
}
}
}
func main() {
if len(os.Args) == 1 {
fmt.Fprintf(os.Stderr, "Usage:\n\t%s [URL]\n", filepath.Base(os.Args[0]))
os.Exit(2)
}
feed, err := rss.Fetch(os.Args[1])
if err != nil {
panic(err)
}
raw, err := json.Marshal(feed)
if err != nil {
panic(err)
}
buf := new(bytes.Buffer)
if err := json.Indent(buf, raw, "", "\t"); err != nil {
panic(err)
}
fmt.Println(buf.String())
}
func addRss(m Message, ws *websocket.Conn) {
feedUrl := getArgument(m, 3, ws)
if feedUrl != "" {
feedUrl = feedUrl[1 : len(feedUrl)-1]
if govalidator.IsURL(feedUrl) {
channelFeeds := feeds[m.Channel]
for _, f := range channelFeeds {
if f.Url == feedUrl {
sendMessage(m.Channel, "Already tracking that url here, no worries homie :snowboarder:", ws)
return
}
}
var newFeed FeedBundle
var err error
newFeed.Url = feedUrl
newFeed.RssHandler, err = rss.Fetch(feedUrl)
if err != nil {
sendMessage(m.Channel, "Incorrect url :OoOoOoOo", ws)
} else {
createNewFeedChecker := false
if _, exists := feeds[m.Channel]; !exists {
createNewFeedChecker = true
}
feeds[m.Channel] = append(feeds[m.Channel], newFeed)
sendMessage(m.Channel, "Yes, sir. "+feedUrl+" added.", ws)
if createNewFeedChecker {
go checkFeed(m.Channel, feeds[m.Channel], ws)
}
}
} else {
sendMessage(m.Channel, "Incorrect url :OoOoOoOo", ws)
}
}
}
func main() {
// Connect to DB
session, err := mgo.Dial("localhost")
if err != nil {
panic(err)
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
db := session.DB("buddio")
avil := db.C("avilable_tracks")
// Get RSS Feed
//feed, err := rss.Fetch("http://feeds.5by5.tv/changelog")
//feed, err := rss.Fetch("http://rss.cnn.com/services/podcasting/newscast/rss.xml")
feed, err := rss.Fetch("http://feeds.kexp.org/kexp/songoftheday")
if err != nil {
panic(err)
}
for _, item := range feed.Items {
err := avil.Insert(&Track{
ID: bson.NewObjectId(),
Title: item.Title,
Length: parseLength(item.Length),
Url: item.Link,
Type: Music,
})
if err != nil {
panic(err)
}
// fmt.Printf("%s - %d\n", item.Length, parseLength(item.Length))
}
// // Get google music tracks list
// client := &http.Client{}
// res, err := client.Do(req)
// if err != nil {
// panic(err)
// }
// defer res.Body.Close()
// body, err := ioutil.ReadAll(res.Body)
// if err != nil {
// panic(err)
// }
// ts := >racks{}
// err = json.Unmarshal(body, &ts)
// if err != nil {
// panic(err)
// }
// // Insert to DB
// for _, gtrack := range ts.Data.Items {
// t := &Track{
// ID: gtrack.ID,
// Title: gtrack.Artist + " - " + gtrack.Title,
// Source: Source{
// Url:
// }
// }
// }
}
func (f Feed) Update() (err error) {
fmt.Printf("Updating feed %v\t%s\n", f.ID, f.Title)
rssfeed, err := rss.Fetch(f.Url)
if err != nil {
glog.Errorf("rss.Fetch(%s): %s", f.Url, err)
f.ErrorString = err.Error()
_ = f.Save()
}
var guid_cache []string
var new_items []Entry
if rssfeed == nil {
return err
}
if len(rssfeed.Items) < 1 {
return err
}
for _, i := range rssfeed.Items {
var e Entry
e.Title = i.Title
skippable, err := f.SkippableEntry(e)
if err != nil {
glog.Errorf("f.SkippableEntry(entry %s): %s", e.Title, err)
skippable = false
}
if skippable {
continue
}
fmt.Printf("%s", i)
e.Content = template.HTML(html.EscapeString(i.Content))
e.Date = fmt.Sprintf("%s", i.Date)
e.FeedID = f.ID
e.GUID = Escape_guid(i.Link)
e.Link = i.Link
e.Marked = "0"
e.Unread = true
guid_cache = append(guid_cache, e.GUID)
new_items = append(new_items, e)
}
existing_entries := make(map[string]Entry)
for _, e := range f.GetEntriesByParam(" guid in ('" + strings.Join(guid_cache, "', '") + "')") {
existing_entries[e.GUID] = e
}
for _, e := range new_items {
if _, ok := existing_entries[e.GUID]; ok {
print(".")
continue
}
skippable, err := f.SkippableEntry(e)
if err != nil {
glog.Errorf("f.SkippableEntry(e): %s", e, err)
continue
}
if skippable {
print(".")
continue
}
print("+")
e.Normalize()
e.Save(f.UserName)
}
print("\n")
f.ClearCache()
return err
}
func ReadFeed(task int, url string, resChan chan *rss.Feed) {
log.Printf("(%03d) start fetch for %s", task, url)
var feed *rss.Feed
file := feedFile(url)
// stat the file so we can tell if it exists
_, err := os.Stat(file)
if err != nil {
log.Printf("(%03d) no cache file, fetching for first time", task)
goto first_fetch
} else {
fd, err := os.Open(file)
if err != nil {
log.Printf("(%03d) could not read cache, refetching: %v", task, err)
goto first_fetch
} else {
log.Printf("(%03d) reading feed cache from %v", task, file)
dec := gob.NewDecoder(fd)
err = dec.Decode(&feed)
fd.Close()
if err != nil {
log.Printf("(%03d) invalid cache file, refetching: %v", task, err)
goto first_fetch
} else {
err = feed.Update()
if err != nil {
log.Printf("(%03d) failed to update: %v", task, err)
resChan <- nil
return
}
goto loaded
}
}
}
first_fetch:
feed, err = rss.Fetch(url)
if err != nil {
log.Printf("(%03d) failed to fetch: %v", task, err)
resChan <- nil
return
}
loaded:
fd, err := os.Create(file)
defer fd.Close()
if err != nil {
log.Println("(%03d) could not create cache file: %v", task, err)
} else {
enc := gob.NewEncoder(fd)
err = enc.Encode(feed)
if err != nil {
log.Println("(%03d) could not write feed cache: %v", task, err)
}
log.Printf("(%03d) updated cache for %s", task, feed.Title)
}
resChan <- feed
}
