/
module_rss.go
157 lines (131 loc) · 3.78 KB
/
module_rss.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
package main
import (
"encoding/json"
"fmt"
"time"
"github.com/nemith/mipples/tinyurl"
"github.com/Sirupsen/logrus"
irc "github.com/fluffle/goirc/client"
rss "github.com/jteeuwen/go-pkg-rss"
)
func init() {
module.Register("rss", &RSSModule{})
}
type RSSFeedConfig struct {
Name string `json:"name"`
URL string `json:"url"`
Channels []string `json:"channels"`
Timeout int `json:"timeout"`
}
type RSSConfig struct {
Timeout int `"json:timeout"`
Feeds []*RSSFeedConfig `json:"feeds"`
}
type RSSModule struct {
config *RSSConfig
}
// RSSLastSeen stores the last RFC seen to be persistant across instances of the
// bot
type RSSLastSeen struct {
Id int
Feed string `sql:"not null;unique"`
Key string `sql:"not null"`
}
func (r RSSLastSeen) TableName() string {
return "rss_lastseen"
}
func (m *RSSModule) Init(c *irc.Conn, config json.RawMessage) {
db.AutoMigrate(&RSSLastSeen{})
err := json.Unmarshal(config, &m.config)
if err != nil {
panic(err)
}
c.HandleFunc(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
// Wait a bit to lets things settle. This should be rewritten
// to check for channel memebership
<-time.After(30 * time.Second)
for _, feed := range m.config.Feeds {
if feed.URL == "" {
log.WithFields(logrus.Fields{
"feed": feed,
}).Error("RSS: Feed has no URL")
}
timeout := feed.Timeout
if timeout < 1 {
if m.config.Timeout < 1 {
log.WithFields(logrus.Fields{
"feed": feed,
}).Error("RSS: Feed has no timeout or global timeout")
}
timeout = m.config.Timeout
}
go pollFeed(c, feed, timeout)
}
})
}
func pollFeed(conn *irc.Conn, feedConfig *RSSFeedConfig, timeout int) {
itemHandler := func(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
tag := feedConfig.Name
if feedConfig.Name == "" {
tag = ch.Title
}
// Dirty reverse
reverseItems := make([]*rss.Item, len(newitems))
for i, item := range newitems {
reverseItems[len(newitems)-i-1] = item
}
rssLastSeen := RSSLastSeen{}
db.Where(RSSLastSeen{Feed: feedConfig.URL}).FirstOrInit(&rssLastSeen)
// This is hack. Testing for gorm.RecordNotFound doesn't seem to be working with
// FirstOrInit
if rssLastSeen.Key == "" {
log.WithFields(logrus.Fields{
"feed": feedConfig.URL,
}).Info("No last seen for feed. Assuming new feed and not spamming the channel")
} else {
for _, item := range reverseItems {
if rssLastSeen.Key == item.Key() {
log.WithFields(logrus.Fields{
"item": item.Title,
"lastseenkey": rssLastSeen.Key,
"item_key": item.Key(),
}).Debug("RSS: Already seen this RSS item")
break
}
// Tinyify item's url
shortURL, err := tinyurl.Tinyify(item.Links[0].Href)
if err != nil {
log.WithFields(logrus.Fields{
"longURL": item.Links[0].Href,
"shortURL": shortURL,
"error": err,
}).Error("RSS: Failed to resolve tinyURL")
shortURL = item.Links[0].Href
}
for _, channel := range feedConfig.Channels {
conn.Privmsg(channel, fmt.Sprintf("[RSS %s] %s - %s", tag, item.Title, shortURL))
}
}
}
// Write the first item to the last seen database
rssLastSeen.Key = reverseItems[0].Key()
db.Save(&rssLastSeen)
}
feed := rss.New(timeout, true, nil, itemHandler)
for {
log.WithFields(logrus.Fields{
"feed": feedConfig.URL,
}).Info("RSS: Fetching rss feed")
if err := feed.Fetch(feedConfig.URL, nil); err != nil {
log.WithFields(logrus.Fields{
"feed": feedConfig.URL,
"error": err,
}).Error("RSS: Error retriving feed.")
}
log.WithFields(logrus.Fields{
"feed": feedConfig.URL,
"secondsTillUpdate": feed.SecondsTillUpdate(),
}).Info("RSS: Waiting")
<-time.After(time.Duration(feed.SecondsTillUpdate()) * time.Second)
}
}